diff --git a/results/trial_1/all-in-one.log b/results/trial_1/all-in-one.log new file mode 100644 index 0000000000000000000000000000000000000000..f1aabfb215fc4b85b416e466e07d50a539bc379c --- /dev/null +++ b/results/trial_1/all-in-one.log @@ -0,0 +1,273 @@ +[ INFO ] Configuration - Loading /eval.properties FROM /home/students/zimmermann/Courses/ws17/fsem/absinth/WSI-Evaluator/config/eval.properties +[ INFO ] WSIEvaluator - Run started at 22:19 +[ INFO ] Dataset - Loading the datasets/trial/ +[ INFO ] Dataset - Loading the topics data datasets/trial/ +[ INFO ] Dataset - Loading the subtopics data +[ INFO ] Dataset - Loading the snippets data +[ INFO ] Dataset - Loading the relations data +[ INFO ] Dataset - Loading the relations data +[ INFO ] WSIEvaluator - +=================================================== +Starting the evaluation +=================================================== +[ INFO ] WSIEvaluator - +=================================================== +subtopic-recall@K default: 100 +=================================================== +[ WARN ] WSIEvaluator - +============= Query 1 : "soul food" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "soul food" =============== +============ Value of Rand Index = 0.6197979797979798 ================================= +============ Partial average Rand Index = 0.6197979797979798 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = 0.0 ================================= +============ Value of Jaccard Index = 0.6197979797979798 ================================= +============ Partial average of Jaccard Index = 0.6197979797979798 ====== +============ Precision = 0.78, Recall = 0.8387096774193549, F1 = 0.8082901554404146 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 100.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 2 : "the block" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "the block" =============== +============ Value of Rand Index = 0.5680808080808081 ================================= +============ Partial average Rand Index = 0.593939393939394 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = 0.0 ================================= +============ Value of Jaccard Index = 0.5680808080808081 ================================= +============ Partial average of Jaccard Index = 0.593939393939394 ====== +============ Precision = 0.19, Recall = 0.7037037037037037, F1 = 0.2992125984251969 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 100.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 3 : "stephen king" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "stephen king" =============== +============ Value of Rand Index = 0.98 ================================= +============ Partial average Rand Index = 0.7226262626262626 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = 0.0 ================================= +============ Value of Jaccard Index = 0.98 ================================= +============ Partial average of Jaccard Index = 0.7226262626262626 ====== +============ Precision = 0.99, Recall = 1.0, F1 = 0.9949748743718593 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 100.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 4 : "cool water" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "cool water" =============== +============ Value of Rand Index = 0.6034343434343434 ================================= +============ Partial average Rand Index = 0.6928282828282828 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = 0.0 ================================= +============ Value of Jaccard Index = 0.6034343434343434 ================================= +============ Partial average of Jaccard Index = 0.6928282828282828 ====== +============ Precision = 0.21, Recall = 0.84, F1 = 0.3359999999999999 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 100.0 ============ +=========================================================================== + +[ INFO ] WSIEvaluator - + +=========== Final average value of S-recall@K: ============= +1 0.3958 +2 0.5625 +3 0.6250 +4 0.7083 +5 0.7083 +6 0.7083 +7 0.7083 +8 0.7917 +9 0.8542 +10 0.8542 +11 0.8542 +12 0.8542 +13 0.8542 +14 0.8542 +15 0.8542 +16 0.8542 +17 0.8542 +18 0.9167 +19 0.9167 +20 0.9167 +21 0.9167 +22 0.9167 +23 0.9167 +24 0.9167 +25 1.0000 +26 1.0000 +27 1.0000 +28 1.0000 +29 1.0000 +30 1.0000 +31 1.0000 +32 1.0000 +33 1.0000 +34 1.0000 +35 1.0000 +36 1.0000 +37 1.0000 +38 1.0000 +39 1.0000 +40 1.0000 +41 1.0000 +42 1.0000 +43 1.0000 +44 1.0000 +45 1.0000 +46 1.0000 +47 1.0000 +48 1.0000 +49 1.0000 +50 1.0000 +51 1.0000 +52 1.0000 +53 1.0000 +54 1.0000 +55 1.0000 +56 1.0000 +57 1.0000 +58 1.0000 +59 1.0000 +60 1.0000 +61 1.0000 +62 1.0000 +63 1.0000 +64 1.0000 +65 1.0000 +66 1.0000 +67 1.0000 +68 1.0000 +69 1.0000 +70 1.0000 +71 1.0000 +72 1.0000 +73 1.0000 +74 1.0000 +75 1.0000 +76 1.0000 +77 1.0000 +78 1.0000 +79 1.0000 +80 1.0000 +81 1.0000 +82 1.0000 +83 1.0000 +84 1.0000 +85 1.0000 +86 1.0000 +87 1.0000 +88 1.0000 +89 1.0000 +90 1.0000 +91 1.0000 +92 1.0000 +93 1.0000 +94 1.0000 +95 1.0000 +96 1.0000 +97 1.0000 +98 1.0000 +99 1.0000 +100 1.0000 + +=========== Final average value of S-precision@r: ============= +0.4000 0.7222 +0.4500 0.5222 +0.5000 0.4111 +0.5500 0.3173 +0.6000 0.3021 +0.6500 0.2646 +0.7000 0.2032 +0.7500 0.1838 +0.8000 0.1905 +0.8500 0.1595 +0.9000 0.1498 +0.9500 0.1413 +1.0000 0.0267 + +=========== Final average value of F1: ===================== +average F1 = 0.6609591661520388 + +=========== Final average value of Rand Index: ============= +average Rand Index = 0.6928282828282828 + +=========== Final average value of Adjusted Rand Index: ==== +average Adj Rand Index = 0.0 + +=========== Final average value of Jaccard Index: ========== +average Jaccard Index = 0.6928282828282828 + +================ Statistics: ==================================== +============ average number of created clusters: 1.0 +============ average cluster size: 100.0 + +[ WARN ] WSIEvaluator - Simulation started at: 22:19 and completed at 22:19 +Execution time: 0.414 sec diff --git a/results/trial_1/baseline.log b/results/trial_1/baseline.log new file mode 100644 index 0000000000000000000000000000000000000000..bfea066b5bd627f2630e82920871c92d952a56bc --- /dev/null +++ b/results/trial_1/baseline.log @@ -0,0 +1,292 @@ +[ INFO ] Configuration - Loading /eval.properties FROM /home/students/zimmermann/Courses/ws17/fsem/absinth/WSI-Evaluator/config/eval.properties +[ INFO ] WSIEvaluator - Run started at 22:02 +[ INFO ] Dataset - Loading the datasets/trial/ +[ INFO ] Dataset - Loading the topics data datasets/trial/ +[ INFO ] Dataset - Loading the subtopics data +[ INFO ] Dataset - Loading the snippets data +[ INFO ] Dataset - Loading the relations data +[ INFO ] Dataset - Loading the relations data +[ INFO ] WSIEvaluator - +=================================================== +Starting the evaluation +=================================================== +[ INFO ] WSIEvaluator - +=================================================== +subtopic-recall@K default: 100 +=================================================== +[ WARN ] WSIEvaluator - +============= Query 1 : "soul food" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 7 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 4, 5, 6, 9, 11, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 29, 30, 32, 33, 35, 36, 38, 39, 40, 42, 43, 44, 45, 47, 49, 50, 51, 57, 58, 59, 61, 63, 64, 65, 66, 68, 70, 71, 73, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 100] +The cluster 5 contains the snippets: [2, 8, 10, 31, 37, 46, 72, 74, 96] +The cluster 3 contains the snippets: [3, 41, 52, 53, 55, 82] +The cluster 4 contains the snippets: [7, 20, 28, 67, 69, 75] +The cluster 6 contains the snippets: [17, 48, 56, 60, 76] +The cluster 7 contains the snippets: [54, 62, 99] +The cluster 2 contains the snippets: [34] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 34, 3, 7, 2, 17, 54, 4, 41, 20, 8, 48, 62, 5, 52, 28, 10, 56, 99, 6, 53, 67, 31, 60, 9, 55, 69, 37, 76, 11, 82, 75, 46, 12, 72, 13, 74, 14, 96, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 29, 30, 32, 33, 35, 36, 38, 39, 40, 42, 43, 44, 45, 47, 49, 50, 51, 57, 58, 59, 61, 63, 64, 65, 66, 68, 70, 71, 73, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "soul food" =============== +============ Value of Rand Index = 0.4218181818181818 ================================= +============ Partial average Rand Index = 0.4218181818181818 ====== +============ Value of Adjusted Rand Index = -0.13342828523132788 ================================= +============ Partial average of Adjusted Rand Index = -0.13342828523132788 ================================= +============ Value of Jaccard Index = 0.32051282051282054 ================================= +============ Partial average of Jaccard Index = 0.32051282051282054 ====== +============ Precision = 0.79, Recall = 0.8494623655913979, F1 = 0.8186528497409327 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 7 ============ +============ average size of the created clusters: 14.285714285714286 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 2 : "the block" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 5 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 80, 81, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 96, 98, 99, 100] +The cluster 2 contains the snippets: [12, 47, 53, 67, 75, 79, 82, 97] +The cluster 3 contains the snippets: [85, 95] +The cluster 4 contains the snippets: [34] +The cluster 5 contains the snippets: [16] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 12, 85, 34, 16, 2, 47, 95, 3, 53, 4, 67, 5, 75, 6, 79, 7, 82, 8, 97, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 80, 81, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 96, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "the block" =============== +============ Value of Rand Index = 0.6139393939393939 ================================= +============ Partial average Rand Index = 0.5178787878787878 ====== +============ Value of Adjusted Rand Index = 0.1643434476034962 ================================= +============ Partial average of Adjusted Rand Index = 0.01545758118608416 ================================= +============ Value of Jaccard Index = 0.5545454545454546 ================================= +============ Partial average of Jaccard Index = 0.4375291375291376 ====== +============ Precision = 0.19, Recall = 0.7037037037037037, F1 = 0.2992125984251969 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 5 ============ +============ average size of the created clusters: 20.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 3 : "stephen king" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 9 snippet clusters: ==================== +The cluster 1 contains the snippets: [2, 5, 10, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 35, 37, 42, 43, 44, 48, 50, 51, 52, 53, 55, 58, 61, 64, 65, 66, 67, 68, 69, 70, 74, 75, 76, 77, 80, 81, 83, 84, 86, 87, 88, 90, 91, 92, 94, 95, 96, 97] +The cluster 4 contains the snippets: [4, 9, 13, 39, 40, 45, 46, 49, 60, 82, 100] +The cluster 8 contains the snippets: [15, 32, 54, 63, 71, 72, 79, 89, 99] +The cluster 2 contains the snippets: [3, 27, 31, 36, 38, 41, 47, 56] +The cluster 6 contains the snippets: [1, 11, 33, 59, 62] +The cluster 5 contains the snippets: [8, 12, 73, 78] +The cluster 9 contains the snippets: [6, 57, 85, 93] +The cluster 3 contains the snippets: [7, 34] +The cluster 7 contains the snippets: [98] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [2, 3, 7, 4, 8, 1, 98, 15, 6, 5, 27, 34, 9, 12, 11, 32, 57, 10, 31, 13, 73, 33, 54, 85, 14, 36, 39, 78, 59, 63, 93, 16, 38, 40, 62, 71, 17, 41, 45, 72, 18, 47, 46, 79, 19, 56, 49, 89, 20, 60, 99, 21, 82, 22, 100, 23, 24, 25, 26, 28, 29, 30, 35, 37, 42, 43, 44, 48, 50, 51, 52, 53, 55, 58, 61, 64, 65, 66, 67, 68, 69, 70, 74, 75, 76, 77, 80, 81, 83, 84, 86, 87, 88, 90, 91, 92, 94, 95, 96, 97] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "stephen king" =============== +============ Value of Rand Index = 0.3375757575757576 ================================= +============ Partial average Rand Index = 0.4577777777777778 ====== +============ Value of Adjusted Rand Index = -0.013200341132411225 ================================= +============ Partial average of Adjusted Rand Index = 0.005904940413252365 ================================= +============ Value of Jaccard Index = 0.33163473298002444 ================================= +============ Partial average of Jaccard Index = 0.4022310026794332 ====== +============ Precision = 0.99, Recall = 1.0, F1 = 0.9949748743718593 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 9 ============ +============ average size of the created clusters: 11.11111111111111 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 4 : "cool water" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 2 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +The cluster 2 contains the snippets: [14, 54] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 14, 2, 54, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "cool water" =============== +============ Value of Rand Index = 0.6050505050505051 ================================= +============ Partial average Rand Index = 0.4945959595959596 ====== +============ Value of Adjusted Rand Index = 0.02423016804345094 ================================= +============ Partial average of Adjusted Rand Index = 0.010486247320802009 ================================= +============ Value of Jaccard Index = 0.5967409240924092 ================================= +============ Partial average of Jaccard Index = 0.45085848303267717 ====== +============ Precision = 0.21, Recall = 0.84, F1 = 0.3359999999999999 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 2 ============ +============ average size of the created clusters: 50.0 ============ +=========================================================================== + +[ INFO ] WSIEvaluator - + +=========== Final average value of S-recall@K: ============= +1 0.3958 +2 0.6250 +3 0.6250 +4 0.6250 +5 0.6250 +6 0.7083 +7 0.7083 +8 0.7083 +9 0.7083 +10 0.7917 +11 0.7917 +12 0.7917 +13 0.7917 +14 0.7917 +15 0.7917 +16 0.7917 +17 0.7917 +18 0.7917 +19 0.7917 +20 0.7917 +21 0.7917 +22 0.7917 +23 0.7917 +24 0.7917 +25 0.8542 +26 0.8542 +27 0.8542 +28 0.8542 +29 0.8542 +30 0.8542 +31 0.8542 +32 0.8542 +33 0.8542 +34 0.8542 +35 0.9375 +36 0.9375 +37 0.9375 +38 0.9375 +39 0.9375 +40 0.9375 +41 0.9375 +42 1.0000 +43 1.0000 +44 1.0000 +45 1.0000 +46 1.0000 +47 1.0000 +48 1.0000 +49 1.0000 +50 1.0000 +51 1.0000 +52 1.0000 +53 1.0000 +54 1.0000 +55 1.0000 +56 1.0000 +57 1.0000 +58 1.0000 +59 1.0000 +60 1.0000 +61 1.0000 +62 1.0000 +63 1.0000 +64 1.0000 +65 1.0000 +66 1.0000 +67 1.0000 +68 1.0000 +69 1.0000 +70 1.0000 +71 1.0000 +72 1.0000 +73 1.0000 +74 1.0000 +75 1.0000 +76 1.0000 +77 1.0000 +78 1.0000 +79 1.0000 +80 1.0000 +81 1.0000 +82 1.0000 +83 1.0000 +84 1.0000 +85 1.0000 +86 1.0000 +87 1.0000 +88 1.0000 +89 1.0000 +90 1.0000 +91 1.0000 +92 1.0000 +93 1.0000 +94 1.0000 +95 1.0000 +96 1.0000 +97 1.0000 +98 1.0000 +99 1.0000 +100 1.0000 + +=========== Final average value of S-precision@r: ============= +0.4000 0.6667 +0.4500 0.4841 +0.5000 0.3611 +0.5500 0.2481 +0.6000 0.2372 +0.6500 0.2091 +0.7000 0.1464 +0.7500 0.1336 +0.8000 0.1249 +0.8500 0.1175 +0.9000 0.1111 +0.9500 0.1054 +1.0000 0.0267 + +=========== Final average value of F1: ===================== +average F1 = 0.6636355295257614 + +=========== Final average value of Rand Index: ============= +average Rand Index = 0.4945959595959596 + +=========== Final average value of Adjusted Rand Index: ==== +average Adj Rand Index = 0.010486247320802009 + +=========== Final average value of Jaccard Index: ========== +average Jaccard Index = 0.45085848303267717 + +================ Statistics: ==================================== +============ average number of created clusters: 5.75 +============ average cluster size: 23.849206349206348 + +[ WARN ] WSIEvaluator - Simulation started at: 22:02 and completed at 22:02 +Execution time: 0.503 sec diff --git a/results/trial_1/singletons.log b/results/trial_1/singletons.log new file mode 100644 index 0000000000000000000000000000000000000000..aa149801df65e7e1acff08b5e8bc5dfde3f2cdb4 --- /dev/null +++ b/results/trial_1/singletons.log @@ -0,0 +1,669 @@ +[ INFO ] Configuration - Loading /eval.properties FROM /home/students/zimmermann/Courses/ws17/fsem/absinth/WSI-Evaluator/config/eval.properties +[ INFO ] WSIEvaluator - Run started at 22:17 +[ INFO ] Dataset - Loading the datasets/trial/ +[ INFO ] Dataset - Loading the topics data datasets/trial/ +[ INFO ] Dataset - Loading the subtopics data +[ INFO ] Dataset - Loading the snippets data +[ INFO ] Dataset - Loading the relations data +[ INFO ] Dataset - Loading the relations data +[ INFO ] WSIEvaluator - +=================================================== +Starting the evaluation +=================================================== +[ INFO ] WSIEvaluator - +=================================================== +subtopic-recall@K default: 100 +=================================================== +[ WARN ] WSIEvaluator - +============= Query 1 : "soul food" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 100 snippet clusters: ==================== +The cluster 1 contains the snippets: [1] +The cluster 2 contains the snippets: [2] +The cluster 3 contains the snippets: [3] +The cluster 4 contains the snippets: [4] +The cluster 5 contains the snippets: [5] +The cluster 6 contains the snippets: [6] +The cluster 7 contains the snippets: [7] +The cluster 8 contains the snippets: [8] +The cluster 9 contains the snippets: [9] +The cluster 10 contains the snippets: [10] +The cluster 11 contains the snippets: [11] +The cluster 12 contains the snippets: [12] +The cluster 13 contains the snippets: [13] +The cluster 14 contains the snippets: [14] +The cluster 15 contains the snippets: [15] +The cluster 16 contains the snippets: [16] +The cluster 17 contains the snippets: [17] +The cluster 18 contains the snippets: [18] +The cluster 19 contains the snippets: [19] +The cluster 20 contains the snippets: [20] +The cluster 21 contains the snippets: [21] +The cluster 22 contains the snippets: [22] +The cluster 23 contains the snippets: [23] +The cluster 24 contains the snippets: [24] +The cluster 25 contains the snippets: [25] +The cluster 26 contains the snippets: [26] +The cluster 27 contains the snippets: [27] +The cluster 28 contains the snippets: [28] +The cluster 29 contains the snippets: [29] +The cluster 30 contains the snippets: [30] +The cluster 31 contains the snippets: [31] +The cluster 32 contains the snippets: [32] +The cluster 33 contains the snippets: [33] +The cluster 34 contains the snippets: [34] +The cluster 35 contains the snippets: [35] +The cluster 36 contains the snippets: [36] +The cluster 37 contains the snippets: [37] +The cluster 38 contains the snippets: [38] +The cluster 39 contains the snippets: [39] +The cluster 40 contains the snippets: [40] +The cluster 41 contains the snippets: [41] +The cluster 42 contains the snippets: [42] +The cluster 43 contains the snippets: [43] +The cluster 44 contains the snippets: [44] +The cluster 45 contains the snippets: [45] +The cluster 46 contains the snippets: [46] +The cluster 47 contains the snippets: [47] +The cluster 48 contains the snippets: [48] +The cluster 49 contains the snippets: [49] +The cluster 50 contains the snippets: [50] +The cluster 51 contains the snippets: [51] +The cluster 52 contains the snippets: [52] +The cluster 53 contains the snippets: [53] +The cluster 54 contains the snippets: [54] +The cluster 55 contains the snippets: [55] +The cluster 56 contains the snippets: [56] +The cluster 57 contains the snippets: [57] +The cluster 58 contains the snippets: [58] +The cluster 59 contains the snippets: [59] +The cluster 60 contains the snippets: [60] +The cluster 61 contains the snippets: [61] +The cluster 62 contains the snippets: [62] +The cluster 63 contains the snippets: [63] +The cluster 64 contains the snippets: [64] +The cluster 65 contains the snippets: [65] +The cluster 66 contains the snippets: [66] +The cluster 67 contains the snippets: [67] +The cluster 68 contains the snippets: [68] +The cluster 69 contains the snippets: [69] +The cluster 70 contains the snippets: [70] +The cluster 71 contains the snippets: [71] +The cluster 72 contains the snippets: [72] +The cluster 73 contains the snippets: [73] +The cluster 74 contains the snippets: [74] +The cluster 75 contains the snippets: [75] +The cluster 76 contains the snippets: [76] +The cluster 77 contains the snippets: [77] +The cluster 78 contains the snippets: [78] +The cluster 79 contains the snippets: [79] +The cluster 80 contains the snippets: [80] +The cluster 81 contains the snippets: [81] +The cluster 82 contains the snippets: [82] +The cluster 83 contains the snippets: [83] +The cluster 84 contains the snippets: [84] +The cluster 85 contains the snippets: [85] +The cluster 86 contains the snippets: [86] +The cluster 87 contains the snippets: [87] +The cluster 88 contains the snippets: [88] +The cluster 89 contains the snippets: [89] +The cluster 90 contains the snippets: [90] +The cluster 91 contains the snippets: [91] +The cluster 92 contains the snippets: [92] +The cluster 93 contains the snippets: [93] +The cluster 94 contains the snippets: [94] +The cluster 95 contains the snippets: [95] +The cluster 96 contains the snippets: [96] +The cluster 97 contains the snippets: [97] +The cluster 98 contains the snippets: [98] +The cluster 99 contains the snippets: [99] +The cluster 100 contains the snippets: [100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "soul food" =============== +============ Value of Rand Index = 0.3802020202020202 ================================= +============ Partial average Rand Index = 0.3802020202020202 ====== +============ Value of Adjusted Rand Index = -3.960673660500297E-4 ================================= +============ Partial average of Adjusted Rand Index = -3.960673660500297E-4 ================================= +============ Value of Jaccard Index = 0.0 ================================= +============ Partial average of Jaccard Index = 0.0 ====== +============ Precision = 0.92, Recall = 0.989247311827957, F1 = 0.9533678756476685 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 100 ============ +============ average size of the created clusters: 1.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 2 : "the block" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 100 snippet clusters: ==================== +The cluster 1 contains the snippets: [1] +The cluster 2 contains the snippets: [2] +The cluster 3 contains the snippets: [3] +The cluster 4 contains the snippets: [4] +The cluster 5 contains the snippets: [5] +The cluster 6 contains the snippets: [6] +The cluster 7 contains the snippets: [7] +The cluster 8 contains the snippets: [8] +The cluster 9 contains the snippets: [9] +The cluster 10 contains the snippets: [10] +The cluster 11 contains the snippets: [11] +The cluster 12 contains the snippets: [12] +The cluster 13 contains the snippets: [13] +The cluster 14 contains the snippets: [14] +The cluster 15 contains the snippets: [15] +The cluster 16 contains the snippets: [16] +The cluster 17 contains the snippets: [17] +The cluster 18 contains the snippets: [18] +The cluster 19 contains the snippets: [19] +The cluster 20 contains the snippets: [20] +The cluster 21 contains the snippets: [21] +The cluster 22 contains the snippets: [22] +The cluster 23 contains the snippets: [23] +The cluster 24 contains the snippets: [24] +The cluster 25 contains the snippets: [25] +The cluster 26 contains the snippets: [26] +The cluster 27 contains the snippets: [27] +The cluster 28 contains the snippets: [28] +The cluster 29 contains the snippets: [29] +The cluster 30 contains the snippets: [30] +The cluster 31 contains the snippets: [31] +The cluster 32 contains the snippets: [32] +The cluster 33 contains the snippets: [33] +The cluster 34 contains the snippets: [34] +The cluster 35 contains the snippets: [35] +The cluster 36 contains the snippets: [36] +The cluster 37 contains the snippets: [37] +The cluster 38 contains the snippets: [38] +The cluster 39 contains the snippets: [39] +The cluster 40 contains the snippets: [40] +The cluster 41 contains the snippets: [41] +The cluster 42 contains the snippets: [42] +The cluster 43 contains the snippets: [43] +The cluster 44 contains the snippets: [44] +The cluster 45 contains the snippets: [45] +The cluster 46 contains the snippets: [46] +The cluster 47 contains the snippets: [47] +The cluster 48 contains the snippets: [48] +The cluster 49 contains the snippets: [49] +The cluster 50 contains the snippets: [50] +The cluster 51 contains the snippets: [51] +The cluster 52 contains the snippets: [52] +The cluster 53 contains the snippets: [53] +The cluster 54 contains the snippets: [54] +The cluster 55 contains the snippets: [55] +The cluster 56 contains the snippets: [56] +The cluster 57 contains the snippets: [57] +The cluster 58 contains the snippets: [58] +The cluster 59 contains the snippets: [59] +The cluster 60 contains the snippets: [60] +The cluster 61 contains the snippets: [61] +The cluster 62 contains the snippets: [62] +The cluster 63 contains the snippets: [63] +The cluster 64 contains the snippets: [64] +The cluster 65 contains the snippets: [65] +The cluster 66 contains the snippets: [66] +The cluster 67 contains the snippets: [67] +The cluster 68 contains the snippets: [68] +The cluster 69 contains the snippets: [69] +The cluster 70 contains the snippets: [70] +The cluster 71 contains the snippets: [71] +The cluster 72 contains the snippets: [72] +The cluster 73 contains the snippets: [73] +The cluster 74 contains the snippets: [74] +The cluster 75 contains the snippets: [75] +The cluster 76 contains the snippets: [76] +The cluster 77 contains the snippets: [77] +The cluster 78 contains the snippets: [78] +The cluster 79 contains the snippets: [79] +The cluster 80 contains the snippets: [80] +The cluster 81 contains the snippets: [81] +The cluster 82 contains the snippets: [82] +The cluster 83 contains the snippets: [83] +The cluster 84 contains the snippets: [84] +The cluster 85 contains the snippets: [85] +The cluster 86 contains the snippets: [86] +The cluster 87 contains the snippets: [87] +The cluster 88 contains the snippets: [88] +The cluster 89 contains the snippets: [89] +The cluster 90 contains the snippets: [90] +The cluster 91 contains the snippets: [91] +The cluster 92 contains the snippets: [92] +The cluster 93 contains the snippets: [93] +The cluster 94 contains the snippets: [94] +The cluster 95 contains the snippets: [95] +The cluster 96 contains the snippets: [96] +The cluster 97 contains the snippets: [97] +The cluster 98 contains the snippets: [98] +The cluster 99 contains the snippets: [99] +The cluster 100 contains the snippets: [100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "the block" =============== +============ Value of Rand Index = 0.4319191919191919 ================================= +============ Partial average Rand Index = 0.40606060606060607 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = -1.9803368302501485E-4 ================================= +============ Value of Jaccard Index = 0.0 ================================= +============ Partial average of Jaccard Index = 0.0 ====== +============ Precision = 0.27, Recall = 1.0, F1 = 0.4251968503937008 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 100 ============ +============ average size of the created clusters: 1.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 3 : "stephen king" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 100 snippet clusters: ==================== +The cluster 1 contains the snippets: [1] +The cluster 2 contains the snippets: [2] +The cluster 3 contains the snippets: [3] +The cluster 4 contains the snippets: [4] +The cluster 5 contains the snippets: [5] +The cluster 6 contains the snippets: [6] +The cluster 7 contains the snippets: [7] +The cluster 8 contains the snippets: [8] +The cluster 9 contains the snippets: [9] +The cluster 10 contains the snippets: [10] +The cluster 11 contains the snippets: [11] +The cluster 12 contains the snippets: [12] +The cluster 13 contains the snippets: [13] +The cluster 14 contains the snippets: [14] +The cluster 15 contains the snippets: [15] +The cluster 16 contains the snippets: [16] +The cluster 17 contains the snippets: [17] +The cluster 18 contains the snippets: [18] +The cluster 19 contains the snippets: [19] +The cluster 20 contains the snippets: [20] +The cluster 21 contains the snippets: [21] +The cluster 22 contains the snippets: [22] +The cluster 23 contains the snippets: [23] +The cluster 24 contains the snippets: [24] +The cluster 25 contains the snippets: [25] +The cluster 26 contains the snippets: [26] +The cluster 27 contains the snippets: [27] +The cluster 28 contains the snippets: [28] +The cluster 29 contains the snippets: [29] +The cluster 30 contains the snippets: [30] +The cluster 31 contains the snippets: [31] +The cluster 32 contains the snippets: [32] +The cluster 33 contains the snippets: [33] +The cluster 34 contains the snippets: [34] +The cluster 35 contains the snippets: [35] +The cluster 36 contains the snippets: [36] +The cluster 37 contains the snippets: [37] +The cluster 38 contains the snippets: [38] +The cluster 39 contains the snippets: [39] +The cluster 40 contains the snippets: [40] +The cluster 41 contains the snippets: [41] +The cluster 42 contains the snippets: [42] +The cluster 43 contains the snippets: [43] +The cluster 44 contains the snippets: [44] +The cluster 45 contains the snippets: [45] +The cluster 46 contains the snippets: [46] +The cluster 47 contains the snippets: [47] +The cluster 48 contains the snippets: [48] +The cluster 49 contains the snippets: [49] +The cluster 50 contains the snippets: [50] +The cluster 51 contains the snippets: [51] +The cluster 52 contains the snippets: [52] +The cluster 53 contains the snippets: [53] +The cluster 54 contains the snippets: [54] +The cluster 55 contains the snippets: [55] +The cluster 56 contains the snippets: [56] +The cluster 57 contains the snippets: [57] +The cluster 58 contains the snippets: [58] +The cluster 59 contains the snippets: [59] +The cluster 60 contains the snippets: [60] +The cluster 61 contains the snippets: [61] +The cluster 62 contains the snippets: [62] +The cluster 63 contains the snippets: [63] +The cluster 64 contains the snippets: [64] +The cluster 65 contains the snippets: [65] +The cluster 66 contains the snippets: [66] +The cluster 67 contains the snippets: [67] +The cluster 68 contains the snippets: [68] +The cluster 69 contains the snippets: [69] +The cluster 70 contains the snippets: [70] +The cluster 71 contains the snippets: [71] +The cluster 72 contains the snippets: [72] +The cluster 73 contains the snippets: [73] +The cluster 74 contains the snippets: [74] +The cluster 75 contains the snippets: [75] +The cluster 76 contains the snippets: [76] +The cluster 77 contains the snippets: [77] +The cluster 78 contains the snippets: [78] +The cluster 79 contains the snippets: [79] +The cluster 80 contains the snippets: [80] +The cluster 81 contains the snippets: [81] +The cluster 82 contains the snippets: [82] +The cluster 83 contains the snippets: [83] +The cluster 84 contains the snippets: [84] +The cluster 85 contains the snippets: [85] +The cluster 86 contains the snippets: [86] +The cluster 87 contains the snippets: [87] +The cluster 88 contains the snippets: [88] +The cluster 89 contains the snippets: [89] +The cluster 90 contains the snippets: [90] +The cluster 91 contains the snippets: [91] +The cluster 92 contains the snippets: [92] +The cluster 93 contains the snippets: [93] +The cluster 94 contains the snippets: [94] +The cluster 95 contains the snippets: [95] +The cluster 96 contains the snippets: [96] +The cluster 97 contains the snippets: [97] +The cluster 98 contains the snippets: [98] +The cluster 99 contains the snippets: [99] +The cluster 100 contains the snippets: [100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "stephen king" =============== +============ Value of Rand Index = 0.02 ================================= +============ Partial average Rand Index = 0.2773737373737374 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = -1.320224553500099E-4 ================================= +============ Value of Jaccard Index = 0.0 ================================= +============ Partial average of Jaccard Index = 0.0 ====== +============ Precision = 0.99, Recall = 1.0, F1 = 0.9949748743718593 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 100 ============ +============ average size of the created clusters: 1.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 4 : "cool water" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 100 snippet clusters: ==================== +The cluster 1 contains the snippets: [1] +The cluster 2 contains the snippets: [2] +The cluster 3 contains the snippets: [3] +The cluster 4 contains the snippets: [4] +The cluster 5 contains the snippets: [5] +The cluster 6 contains the snippets: [6] +The cluster 7 contains the snippets: [7] +The cluster 8 contains the snippets: [8] +The cluster 9 contains the snippets: [9] +The cluster 10 contains the snippets: [10] +The cluster 11 contains the snippets: [11] +The cluster 12 contains the snippets: [12] +The cluster 13 contains the snippets: [13] +The cluster 14 contains the snippets: [14] +The cluster 15 contains the snippets: [15] +The cluster 16 contains the snippets: [16] +The cluster 17 contains the snippets: [17] +The cluster 18 contains the snippets: [18] +The cluster 19 contains the snippets: [19] +The cluster 20 contains the snippets: [20] +The cluster 21 contains the snippets: [21] +The cluster 22 contains the snippets: [22] +The cluster 23 contains the snippets: [23] +The cluster 24 contains the snippets: [24] +The cluster 25 contains the snippets: [25] +The cluster 26 contains the snippets: [26] +The cluster 27 contains the snippets: [27] +The cluster 28 contains the snippets: [28] +The cluster 29 contains the snippets: [29] +The cluster 30 contains the snippets: [30] +The cluster 31 contains the snippets: [31] +The cluster 32 contains the snippets: [32] +The cluster 33 contains the snippets: [33] +The cluster 34 contains the snippets: [34] +The cluster 35 contains the snippets: [35] +The cluster 36 contains the snippets: [36] +The cluster 37 contains the snippets: [37] +The cluster 38 contains the snippets: [38] +The cluster 39 contains the snippets: [39] +The cluster 40 contains the snippets: [40] +The cluster 41 contains the snippets: [41] +The cluster 42 contains the snippets: [42] +The cluster 43 contains the snippets: [43] +The cluster 44 contains the snippets: [44] +The cluster 45 contains the snippets: [45] +The cluster 46 contains the snippets: [46] +The cluster 47 contains the snippets: [47] +The cluster 48 contains the snippets: [48] +The cluster 49 contains the snippets: [49] +The cluster 50 contains the snippets: [50] +The cluster 51 contains the snippets: [51] +The cluster 52 contains the snippets: [52] +The cluster 53 contains the snippets: [53] +The cluster 54 contains the snippets: [54] +The cluster 55 contains the snippets: [55] +The cluster 56 contains the snippets: [56] +The cluster 57 contains the snippets: [57] +The cluster 58 contains the snippets: [58] +The cluster 59 contains the snippets: [59] +The cluster 60 contains the snippets: [60] +The cluster 61 contains the snippets: [61] +The cluster 62 contains the snippets: [62] +The cluster 63 contains the snippets: [63] +The cluster 64 contains the snippets: [64] +The cluster 65 contains the snippets: [65] +The cluster 66 contains the snippets: [66] +The cluster 67 contains the snippets: [67] +The cluster 68 contains the snippets: [68] +The cluster 69 contains the snippets: [69] +The cluster 70 contains the snippets: [70] +The cluster 71 contains the snippets: [71] +The cluster 72 contains the snippets: [72] +The cluster 73 contains the snippets: [73] +The cluster 74 contains the snippets: [74] +The cluster 75 contains the snippets: [75] +The cluster 76 contains the snippets: [76] +The cluster 77 contains the snippets: [77] +The cluster 78 contains the snippets: [78] +The cluster 79 contains the snippets: [79] +The cluster 80 contains the snippets: [80] +The cluster 81 contains the snippets: [81] +The cluster 82 contains the snippets: [82] +The cluster 83 contains the snippets: [83] +The cluster 84 contains the snippets: [84] +The cluster 85 contains the snippets: [85] +The cluster 86 contains the snippets: [86] +The cluster 87 contains the snippets: [87] +The cluster 88 contains the snippets: [88] +The cluster 89 contains the snippets: [89] +The cluster 90 contains the snippets: [90] +The cluster 91 contains the snippets: [91] +The cluster 92 contains the snippets: [92] +The cluster 93 contains the snippets: [93] +The cluster 94 contains the snippets: [94] +The cluster 95 contains the snippets: [95] +The cluster 96 contains the snippets: [96] +The cluster 97 contains the snippets: [97] +The cluster 98 contains the snippets: [98] +The cluster 99 contains the snippets: [99] +The cluster 100 contains the snippets: [100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "cool water" =============== +============ Value of Rand Index = 0.39656565656565657 ================================= +============ Partial average Rand Index = 0.3071717171717172 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = -9.901684151250742E-5 ================================= +============ Value of Jaccard Index = 0.0 ================================= +============ Partial average of Jaccard Index = 0.0 ====== +============ Precision = 0.25, Recall = 1.0, F1 = 0.4 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 100 ============ +============ average size of the created clusters: 1.0 ============ +=========================================================================== + +[ INFO ] WSIEvaluator - + +=========== Final average value of S-recall@K: ============= +1 0.3958 +2 0.5625 +3 0.6250 +4 0.7083 +5 0.7083 +6 0.7083 +7 0.7083 +8 0.7917 +9 0.8542 +10 0.8542 +11 0.8542 +12 0.8542 +13 0.8542 +14 0.8542 +15 0.8542 +16 0.8542 +17 0.8542 +18 0.9167 +19 0.9167 +20 0.9167 +21 0.9167 +22 0.9167 +23 0.9167 +24 0.9167 +25 1.0000 +26 1.0000 +27 1.0000 +28 1.0000 +29 1.0000 +30 1.0000 +31 1.0000 +32 1.0000 +33 1.0000 +34 1.0000 +35 1.0000 +36 1.0000 +37 1.0000 +38 1.0000 +39 1.0000 +40 1.0000 +41 1.0000 +42 1.0000 +43 1.0000 +44 1.0000 +45 1.0000 +46 1.0000 +47 1.0000 +48 1.0000 +49 1.0000 +50 1.0000 +51 1.0000 +52 1.0000 +53 1.0000 +54 1.0000 +55 1.0000 +56 1.0000 +57 1.0000 +58 1.0000 +59 1.0000 +60 1.0000 +61 1.0000 +62 1.0000 +63 1.0000 +64 1.0000 +65 1.0000 +66 1.0000 +67 1.0000 +68 1.0000 +69 1.0000 +70 1.0000 +71 1.0000 +72 1.0000 +73 1.0000 +74 1.0000 +75 1.0000 +76 1.0000 +77 1.0000 +78 1.0000 +79 1.0000 +80 1.0000 +81 1.0000 +82 1.0000 +83 1.0000 +84 1.0000 +85 1.0000 +86 1.0000 +87 1.0000 +88 1.0000 +89 1.0000 +90 1.0000 +91 1.0000 +92 1.0000 +93 1.0000 +94 1.0000 +95 1.0000 +96 1.0000 +97 1.0000 +98 1.0000 +99 1.0000 +100 1.0000 + +=========== Final average value of S-precision@r: ============= +0.4000 0.7222 +0.4500 0.5222 +0.5000 0.4111 +0.5500 0.3173 +0.6000 0.3021 +0.6500 0.2646 +0.7000 0.2032 +0.7500 0.1838 +0.8000 0.1905 +0.8500 0.1595 +0.9000 0.1498 +0.9500 0.1413 +1.0000 0.0267 + +=========== Final average value of F1: ===================== +average F1 = 0.7550628988760281 + +=========== Final average value of Rand Index: ============= +average Rand Index = 0.3071717171717172 + +=========== Final average value of Adjusted Rand Index: ==== +average Adj Rand Index = -9.901684151250742E-5 + +=========== Final average value of Jaccard Index: ========== +average Jaccard Index = 0.0 + +================ Statistics: ==================================== +============ average number of created clusters: 100.0 +============ average cluster size: 1.0 + +[ WARN ] WSIEvaluator - Simulation started at: 22:17 and completed at 22:17 +Execution time: 0.552 sec diff --git a/results/trial_1/system.log b/results/trial_1/system.log new file mode 100644 index 0000000000000000000000000000000000000000..79428aab4d62316b57244d661c565107494d4598 --- /dev/null +++ b/results/trial_1/system.log @@ -0,0 +1,283 @@ +[ INFO ] Configuration - Loading /eval.properties FROM /home/students/zimmermann/Courses/ws17/fsem/absinth/WSI-Evaluator/config/eval.properties +[ INFO ] WSIEvaluator - Run started at 22:20 +[ INFO ] Dataset - Loading the datasets/trial/ +[ INFO ] Dataset - Loading the topics data datasets/trial/ +[ INFO ] Dataset - Loading the subtopics data +[ INFO ] Dataset - Loading the snippets data +[ INFO ] Dataset - Loading the relations data +[ INFO ] Dataset - Loading the relations data +[ INFO ] WSIEvaluator - +=================================================== +Starting the evaluation +=================================================== +[ INFO ] WSIEvaluator - +=================================================== +subtopic-recall@K default: 100 +=================================================== +[ WARN ] WSIEvaluator - +============= Query 1 : "soul food" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 7, 8, 10, 12, 14, 17, 20, 28, 31, 33, 34, 37, 42, 46, 48, 49, 52, 55, 56, 60, 67, 69, 72, 74, 75, 76, 78, 80, 85, 93, 94, 95, 96, 98, 13, 38, 50, 54, 58, 62, 65, 70, 71, 83, 91, 97, 99] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 7, 8, 10, 12, 14, 17, 20, 28, 31, 33, 34, 37, 42, 46, 48, 49, 52, 55, 56, 60, 67, 69, 72, 74, 75, 76, 78, 80, 85, 93, 94, 95, 96, 98, 13, 38, 50, 54, 58, 62, 65, 70, 71, 83, 91, 97, 99, 6, 9, 11, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 29, 30, 32, 35, 36, 39, 40, 41, 43, 44, 45, 47, 51, 53, 57, 59, 61, 63, 64, 66, 68, 73, 77, 79, 81, 82, 84, 86, 87, 88, 89, 90, 92, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "soul food" =============== +============ Value of Rand Index = 0.5341414141414141 ================================= +============ Partial average Rand Index = 0.5341414141414141 ====== +============ Value of Adjusted Rand Index = 0.06764734161381367 ================================= +============ Partial average of Adjusted Rand Index = 0.06764734161381367 ================================= +============ Value of Jaccard Index = 0.4108329075114972 ================================= +============ Partial average of Jaccard Index = 0.4108329075114972 ====== +============ Precision = 0.9230769230769231, Recall = 0.5161290322580645, F1 = 0.6620689655172414 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 52.0 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 2 : "the block" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 9 snippet clusters: ==================== +The cluster 1 contains the snippets: [4, 8, 11, 13, 14, 19, 23, 24, 25, 30, 32, 33, 35, 36, 42, 44, 45, 46, 48, 49, 50, 51, 54, 55, 56, 57, 58, 61, 62, 65, 69, 70, 72, 74, 80, 86, 89, 90, 93, 95, 96, 97, 99, 17, 63, 64, 81, 87, 88, 98] +The cluster 4 contains the snippets: [2, 12, 47, 53, 66, 67, 75, 79, 82, 85, 91, 94, 6, 21, 22, 29, 40, 76] +The cluster 5 contains the snippets: [3, 9, 26, 27, 38, 60, 73, 77, 15, 59, 84] +The cluster 2 contains the snippets: [16, 20, 31, 43, 52, 28] +The cluster 9 contains the snippets: [71, 78, 83, 92] +The cluster 6 contains the snippets: [7, 18, 39] +The cluster 3 contains the snippets: [37, 68, 100] +The cluster 8 contains the snippets: [5, 41] +The cluster 7 contains the snippets: [34] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [4, 16, 37, 2, 3, 7, 34, 5, 71, 8, 20, 68, 12, 9, 18, 41, 78, 11, 31, 100, 47, 26, 39, 83, 13, 43, 53, 27, 92, 14, 52, 66, 38, 19, 28, 67, 60, 23, 75, 73, 24, 79, 77, 25, 82, 15, 30, 85, 59, 32, 91, 84, 33, 94, 35, 6, 36, 21, 42, 22, 44, 29, 45, 40, 46, 76, 48, 49, 50, 51, 54, 55, 56, 57, 58, 61, 62, 65, 69, 70, 72, 74, 80, 86, 89, 90, 93, 95, 96, 97, 99, 17, 63, 64, 81, 87, 88, 98, 1, 10] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "the block" =============== +============ Value of Rand Index = 0.5361616161616162 ================================= +============ Partial average Rand Index = 0.5351515151515152 ====== +============ Value of Adjusted Rand Index = 0.12129353212290198 ================================= +============ Partial average of Adjusted Rand Index = 0.09447043686835782 ================================= +============ Value of Jaccard Index = 0.3010654490106545 ================================= +============ Partial average of Jaccard Index = 0.35594917826107586 ====== +============ Precision = 0.20408163265306123, Recall = 0.7407407407407407, F1 = 0.32 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 9 ============ +============ average size of the created clusters: 10.88888888888889 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 3 : "stephen king" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 3 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 32, 33, 35, 36, 38, 39, 40, 41, 42, 43, 45, 46, 48, 49, 50, 51, 54, 55, 57, 58, 59, 60, 62, 63, 67, 69, 71, 72, 73, 76, 78, 79, 82, 83, 85, 88, 89, 92, 93, 95, 96, 97, 98, 99, 100, 26, 30, 37, 66, 68, 70, 75, 80, 81, 94] +The cluster 2 contains the snippets: [7, 31, 34, 44, 47, 56, 64, 86, 91, 22, 53, 84, 87] +The cluster 3 contains the snippets: [5, 90] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 7, 5, 2, 31, 90, 3, 34, 4, 44, 6, 47, 8, 56, 9, 64, 10, 86, 11, 91, 12, 22, 13, 53, 14, 84, 15, 87, 16, 17, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 32, 33, 35, 36, 38, 39, 40, 41, 42, 43, 45, 46, 48, 49, 50, 51, 54, 55, 57, 58, 59, 60, 62, 63, 67, 69, 71, 72, 73, 76, 78, 79, 82, 83, 85, 88, 89, 92, 93, 95, 96, 97, 98, 99, 100, 26, 30, 37, 66, 68, 70, 75, 80, 81, 94, 52, 61, 65, 74, 77] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "stephen king" =============== +============ Value of Rand Index = 0.6747474747474748 ================================= +============ Partial average Rand Index = 0.5816835016835017 ====== +============ Value of Adjusted Rand Index = 0.07041733065428765 ================================= +============ Partial average of Adjusted Rand Index = 0.08645273479700111 ================================= +============ Value of Jaccard Index = 0.6683831101956745 ================================= +============ Partial average of Jaccard Index = 0.4600938222392754 ====== +============ Precision = 1.0, Recall = 0.9595959595959596, F1 = 0.979381443298969 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 3 ============ +============ average size of the created clusters: 31.666666666666668 ============ +=========================================================================== + +[ WARN ] WSIEvaluator - +============= Query 4 : "cool water" ============== + +[ INFO ] WSIEvaluator - +============== Cluster creation phase ================== + +[ WARN ] WSIEvaluator - ClusteredSnippets: + +================== 1 snippet clusters: ==================== +The cluster 1 contains the snippets: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + +[ INFO ] WSIEvaluator - +================== Starting Evaluation ================== +Resulting list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] +[ WARN ] WSIEvaluator - +================ Results of Rand Index for the topic "cool water" =============== +============ Value of Rand Index = 0.6034343434343434 ================================= +============ Partial average Rand Index = 0.5871212121212122 ====== +============ Value of Adjusted Rand Index = 0.0 ================================= +============ Partial average of Adjusted Rand Index = 0.06483955109775083 ================================= +============ Value of Jaccard Index = 0.6034343434343434 ================================= +============ Partial average of Jaccard Index = 0.4959289525380424 ====== +============ Precision = 0.21, Recall = 0.84, F1 = 0.3359999999999999 ============ + +[ WARN ] WSIEvaluator - +================ Statistics: ==================================== +============ number of created clusters: 1 ============ +============ average size of the created clusters: 100.0 ============ +=========================================================================== + +[ INFO ] WSIEvaluator - + +=========== Final average value of S-recall@K: ============= +1 0.3125 +2 0.3958 +3 0.4583 +4 0.6250 +5 0.6250 +6 0.6250 +7 0.6250 +8 0.7917 +9 0.7917 +10 0.7917 +11 0.7917 +12 0.7917 +13 0.7917 +14 0.7917 +15 0.7917 +16 0.7917 +17 0.7917 +18 0.7917 +19 0.7917 +20 0.7917 +21 0.7917 +22 0.7917 +23 0.7917 +24 0.7917 +25 0.7917 +26 0.7917 +27 0.7917 +28 0.7917 +29 0.7917 +30 0.7917 +31 0.7917 +32 0.7917 +33 0.7917 +34 0.7917 +35 0.8750 +36 0.8750 +37 0.8750 +38 0.8750 +39 0.8750 +40 0.9375 +41 0.9375 +42 0.9375 +43 0.9375 +44 0.9375 +45 0.9375 +46 0.9375 +47 0.9375 +48 0.9375 +49 0.9375 +50 0.9375 +51 0.9375 +52 0.9375 +53 0.9375 +54 0.9375 +55 0.9375 +56 0.9375 +57 0.9375 +58 1.0000 +59 1.0000 +60 1.0000 +61 1.0000 +62 1.0000 +63 1.0000 +64 1.0000 +65 1.0000 +66 1.0000 +67 1.0000 +68 1.0000 +69 1.0000 +70 1.0000 +71 1.0000 +72 1.0000 +73 1.0000 +74 1.0000 +75 1.0000 +76 1.0000 +77 1.0000 +78 1.0000 +79 1.0000 +80 1.0000 +81 1.0000 +82 1.0000 +83 1.0000 +84 1.0000 +85 1.0000 +86 1.0000 +87 1.0000 +88 1.0000 +89 1.0000 +90 1.0000 +91 1.0000 +92 1.0000 +93 1.0000 +94 1.0000 +95 1.0000 +96 1.0000 +97 1.0000 +98 1.0000 +99 1.0000 +100 1.0000 + +=========== Final average value of S-precision@r: ============= +0.4000 0.6042 +0.4500 0.3741 +0.5000 0.3111 +0.5500 0.1981 +0.6000 0.2037 +0.6500 0.1813 +0.7000 0.1496 +0.7500 0.1375 +0.8000 0.1275 +0.8500 0.1191 +0.9000 0.1119 +0.9500 0.1057 +1.0000 0.0267 + +=========== Final average value of F1: ===================== +average F1 = 0.6622119613237013 + +=========== Final average value of Rand Index: ============= +average Rand Index = 0.5871212121212122 + +=========== Final average value of Adjusted Rand Index: ==== +average Adj Rand Index = 0.06483955109775083 + +=========== Final average value of Jaccard Index: ========== +average Jaccard Index = 0.4959289525380424 + +================ Statistics: ==================================== +============ average number of created clusters: 3.5 +============ average cluster size: 48.638888888888886 + +[ WARN ] WSIEvaluator - Simulation started at: 22:20 and completed at 22:20 +Execution time: 0.385 sec