Reestore WEAT files

WEAT/run_weat.sh

+#!/bin/bash
+#
+#SBATCH --job-name=weat
+#SBATCH --output=weat_output_4.txt
+#SBATCH --mem=32G
+#SBATCH --partition=compute
+#SBATCH --cpus-per-task=32
+#SBATCH --mail-user=reichelt@cl.uni-heidelberg.de
+#SBATCH --mail-type=ALL
+#SBATCH --time=3-00:00:00
+
+# JOB STEPS
+source /home/students/reichelt/ba/bias-mitigation-ba/bias-venv/bin/activate
+
+srun python /home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py --attribute italian --vector_location /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_polish_w2vformat.txt
+srun python /home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py --attribute turkish --vector_location /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_italian_w2vformat.txt
+srun python /home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py --attribute polish --vector_location /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_italian_w2vformat.txt

WEAT/weat_experiments.py

+"""
+A .py version of the WEAT experiments notebook, created so I can
+run the script using SBATCH with various parameters. Limited to
+German options and does not include W2V, unlike notebook.
+
+Parallelizes the permutation test.
+
+Checks bias in various pre-trained embeddings using the WEAT. Different
+embeddings and attribute lists can be chosen.
+"""
+import argparse
+from itertools import combinations, islice
+from sklearn.metrics.pairwise import cosine_similarity as cosine
+import numpy as np
+from gensim.models import KeyedVectors
+from concurrent.futures import ProcessPoolExecutor
+
+
+def get_target_words(version: str) -> dict:
+    """
+    load German target words for origin as per Kurpicz-Briki 2020 (WEAT 5),
+    or alternatively WEAT 6 target words from Caliskan et al 2017
+    """
+    if version == "reproduction":
+        pleasant = {  # aka career
+            "executive": [], "management": [], "professional": [], "corporation": [],
+            "salary": [], "office": [], "business": [], "career": [],
+        }
+        unpleasant = {  # aka family
+            "home": [], "parents": [], "children": [], "family": [], "cousins": [],
+            "marriage": [], "wedding": [], "relatives": [],
+        }
+    else:
+        pleasant = {
+            "spaß": [],
+            "liebe": [],
+            "frieden": [],
+            "wunderbar": [],
+            "freude": [],
+            "lachen": [],
+            "glück": [],
+        }
+        unpleasant = {
+            "qual": [],
+            "furchtbar": [],
+            "schrecklich": [],
+            "übel": [],
+            "böse": [],
+            "krieg": [],
+            "grausam": [],
+            "versagen": []
+        }
+    return {"pleasant": pleasant, "unpleasant": unpleasant}
+
+
+def get_attribute_words(version: str) -> dict:
+    """Load name lists depending on country. Alternatively load version by Kurpicz-Briki"""
+    if version == "kurpicz":
+        german = {
+            "Peter": [], "Daniel": [], "Hans": [], "Thomas": [], "Andreas": [],
+            "Martin": [], "Markus": [], "Michael": [], "Maria": [], "Anna": [],
+            "Ursula": [], "Ruth": [], "Monika": [], "Elisabeth": [], "Verena": [],
+            "Sandra": []
+        }
+
+        foreign = {
+            "Ladina": [], "Fatima": [], "Fatma": [], "Alma": [], "Soraya": [],
+            "Svetlana": [], "Elif": [], "Vesna": [], "Mehmet": [], "Mustafa": [],
+            "Aleksandar": [], "Mohamed": [], "Ibrahim": [], "Dragan": [],
+            "Hasan": [], "Mohammad": []
+        }
+
+        return {"german": german, "foreign": foreign}
+    elif version == "reproduction":
+        german = {  # aka male
+            "John": [], "Paul": [], "Mike": [], "Kevin": [], "Steve": [],
+            "Greg": [], "Jeff": [], "Bill": []
+        }
+        foreign = {  # aka female
+            "Amy": [], "Joan": [], "Lisa": [], "Sarah": [], "Diana": [],
+            "Kate": [], "Ann": [], "Donna": []
+        }
+        return {"german": german, "foreign": foreign}
+    else:
+        german = {
+            "Katharina": [], "Susanne": [], "Karin": [], "Ulrike": [], "Renate": [],
+            "Birgit": [], "Bettina": [], "Jutta": [], "Ute": [], "Cornelia": [],
+            "Katja": [], "Heike": [], "Stefanie": [], "Kerstin": [], "Tanja": [],
+            "Hans": [], "Carl": [], "Wolfgang": [], "Andreas": [], "Werner": [],
+            "Christoph": [], "Klaus": [], "Philipp": [], "Joachim": [], "Jürgen": [],
+            "Dieter": [], "Matthias": [], "Manfred": [], "Sebastian": [], "Rainer": []
+        }
+
+        if version == "turkish":
+            foreign = {
+                "Esra": [], "Merve": [], "Fatma": [], "Sibel": [], "Elif": [], "Ayşe": [],
+                "Emine": [], "Özlem": [], "Zeynep": [], "Hatice": [], "Dilek": [], "Ebru": [],
+                "Pınar": [], "Hülya": [], "Derya": [], "Mustafa": [], "Murat": [],
+                "Ahmet": [], "Kemal": [], "Orhan": [], "Hüseyin": [], "Bülent": [],
+                "Metin": [], "Ömer": [], "Emre": [], "Halil": [], "Erkan": [],
+                "Uğur": [], "Burak": [], "Volkan": []
+            }
+
+        elif version == "polish":
+            foreign = {
+                "Magdalena": [], "Ewa": [], "Zofia": [], "Beata": [], "Katarzyna": [],
+                "Krystyna": [], "Małgorzata": [], "Jadwiga": [], "Danuta": [],
+                "Elżbieta": [], "Urszula": [], "Alicja": [], "Aneta": [], "Iwona": [],
+                "Edyta": [], "Andrzej": [], "Stanisław": [], "Marek": [], "Józef": [],
+                "Henryk": [], "Krzysztof": [], "Władysław": [], "Tadeusz": [], "Piotr": [],
+                "Janusz": [], "Tomasz": [], "Wojciech": [], "Jakub": [], "Marcin": [],
+                "Franciszek": []
+            }
+
+        elif version == "italian":
+            foreign = {
+                "Caterina": [], "Francesca": [], "Paola": [], "Giulia": [], "Chiara": [],
+                "Giovanna": [], "Alessandra": [], "Gioia": [], "Antonella": [],
+                "Giuseppina": [], "Azzurra": [], "Antonietta": [], "Ambra": [],
+                "Alessia": [], "Giorgia": [], "Giovanni": [], "Carlo": [],
+                "Francesco": [], "Giuseppe": [], "Pietro": [], "Luigi": [], "Paolo": [],
+                "Alessandro": [], "Angelo": [], "Giorgio": [], "Domenico": [],
+                "Enrico": [], "Stefano": [], "Vincenzo": [], "Matteo": []
+            }
+
+        else:
+            raise ValueError("Invalid version specified. See --help")
+
+        return {"german": german, "foreign": foreign}
+
+
+def get_embeddings(lookup_dict: dict, embeddings) -> dict():
+    """Go through nested seed dicts and look up embedding for each word"""
+    for category, seeds in lookup_dict.items():
+        for word, _ in seeds.items():
+            if word.lower() in embeddings:
+                seeds[word] = embeddings[word.lower()]
+            else:
+                raise KeyError(f"'{word}' not in vocabulary")
+        lookup_dict[category] = seeds
+    return lookup_dict
+
+
+def attribute_association_s(word_vector, target_set1, target_set2):
+    reshaped_word_vector = np.array(word_vector).reshape(1, -1)
+    sims1 = [cosine(reshaped_word_vector, np.array(vec).reshape(1, -1)) for vec in list(target_set1.values())]
+    sims2 = [cosine(reshaped_word_vector, np.array(vec).reshape(1, -1)) for vec in list(target_set2.values())]
+    return np.mean(sims1) - np.mean(sims2)
+
+
+def differential_association_s(attr1, attr2, target1, target2):
+    sum1 = sum([attribute_association_s(vec, target1, target2) for vec in list(attr1.values())])
+    sum2 = sum([attribute_association_s(vec, target1, target2) for vec in list(attr2.values())])
+    return sum1 - sum2
+
+
+def cohens_d_calc(target1, target2, attr1, attr2):
+    mean1 = np.mean([attribute_association_s(x, attr1, attr2) for x in list(target1.values())])
+    mean2 = np.mean([attribute_association_s(x, attr1, attr2) for x in list(target2.values())])
+    join = list(target1.values()) + (list(target2.values()))
+    joint_association = [attribute_association_s(x, attr1, attr2) for x in join]
+    stddev = np.std(joint_association)
+    return (mean1 - mean2) / stddev
+
+
+def permutations(target1, target2):
+    join = list(target1.keys()) + list(target2.keys())
+    combs = list(islice(combinations(join, int(len(join)/2)), 100000))
+    first_groups = []
+    second_groups = []
+    for c in combs:
+        rest = []
+        for e in join:
+            if e not in c:
+                rest.append(e)
+        first_groups.append(c)
+        second_groups.append(rest)
+    return first_groups, second_groups
+
+
+def p_value_calc_worker(args):
+    X_subset, Y_subset, comparison, attr1, attr2 = args
+    return differential_association_s(X_subset, Y_subset, attr1, attr2) > comparison
+
+def p_value_calc(comparison, X_perms, Y_perms, target1, target2, attr1, attr2):
+    counter = 0
+    joint_dict = {**target1, **target2}
+
+    with ProcessPoolExecutor() as executor:
+        args_list = []
+        for i, _ in enumerate(X_perms):
+            X_subset = {key: joint_dict[key] for key in X_perms[i]}
+            Y_subset = {key: joint_dict[key] for key in Y_perms[i]}
+            args_list.append((X_subset, Y_subset, comparison, attr1, attr2))
+
+        results = list(executor.map(p_value_calc_worker, args_list))
+        counter = sum(results)
+
+    return counter
+
+def calculate_WEAT(target_data: dict, attribute_data: dict) -> tuple:
+    X = attribute_data["german"]
+    Y = attribute_data["foreign"]
+    A = target_data["pleasant"]
+    B = target_data["unpleasant"]
+
+    original_diff_association = differential_association_s(X, Y, A, B)
+    d = cohens_d_calc(X, Y, A, B)
+    X_i, Y_i = permutations(X, Y)
+    p_value_count = p_value_calc(original_diff_association, X_i, Y_i, X, Y, A, B)
+    p = p_value_count/100000
+
+    return d, p
+
+
+
+if __name__ == "__main__":
+    np.random.seed(42)
+
+    parser = argparse.ArgumentParser(
+        description="Calculate WEAT score for given attributes and vectors")
+    parser.add_argument("--attribute", help="'kurpicz', 'turkish', 'polish', 'reproduction'")
+    parser.add_argument("--vector_location", help="specify a file path to embeddings")
+    args = parser.parse_args()
+
+    print("Loading seed words...")
+    target_dicts = get_target_words(args.attribute)
+    attribute_dicts = get_attribute_words(args.attribute)
+
+    print("Loading model...")
+    model = KeyedVectors.load_word2vec_format(args.vector_location, binary=False)
+
+    print("Retrieving embeddings...")
+    target_dicts = get_embeddings(target_dicts, model)
+    attribute_dicts = get_embeddings(attribute_dicts, model)
+
+    print("Calculating WEAT...")
+    cohens_d, p_value = calculate_WEAT(target_dicts, attribute_dicts)
+
+    print(f"WEAT scores for: {args.attribute} test, vectors from {args.vector_location}")
+    print(f"Cohen's d: {cohens_d:.4f}, p-value: {p_value:.4f}")
+    print("-----------------------------------------------")

WEAT/weat_output.txt

+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/wiki.de.vec
+Cohen's d: 1.3571, p-value: 0.0109
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/wiki.de.vec
+Cohen's d: 0.2829, p-value: 0.5185
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/wiki.de.vec
+Cohen's d: 1.0331, p-value: 0.1082
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_turkish_w2vformat.txt
+Cohen's d: 1.1332, p-value: 0.0573
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_polish_w2vformat.txt
+Cohen's d: 0.1786, p-value: 0.5699
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_italian_w2vformat.txt
+Cohen's d: 0.5896, p-value: 0.3510
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_turkish_w2vformat.txt
+Cohen's d: 0.2123, p-value: 0.5639
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_turkish_w2vformat.txt
+Cohen's d: 0.9480, p-value: 0.1469
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_polish_w2vformat.txt
+Cohen's d: 1.3612, p-value: 0.0103
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_polish_w2vformat.txt
+Cohen's d: 1.0477, p-value: 0.1018
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_italian_w2vformat.txt
+Cohen's d: 1.3299, p-value: 0.0139
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/fasttext/fasttext_hard_debiased_italian_w2vformat.txt
+Cohen's d: 0.1901, p-value: 0.5747
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Traceback (most recent call last):
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 212, in <module>
+    target_dicts = get_embeddings(target_dicts, model)
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 117, in get_embeddings
+    raise KeyError(f"'{word}' not in vocabulary")
+KeyError: "'scheußlich' not in vocabulary"
+srun: error: node37: task 0: Exited with exit code 1

WEAT/weat_output_2.txt

+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: kurpicz test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/vectors_no_debiasing.txt
+Cohen's d: 1.7889, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/vectors_no_debiasing.txt
+Cohen's d: 1.8321, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/vectors_no_debiasing.txt
+Cohen's d: 1.4650, p-value: 0.0004
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_turkish.txt
+Cohen's d: 1.8209, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_polish.txt
+Cohen's d: 1.5567, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_italian.txt
+Cohen's d: 1.3057, p-value: 0.0001
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_turkish.txt
+Cohen's d: 1.6582, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_turkish.txt
+Cohen's d: 1.4522, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_polish.txt
+Cohen's d: 1.7717, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_polish.txt
+Cohen's d: 1.3303, p-value: 0.0001
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_italian.txt
+Cohen's d: 1.7891, p-value: 0.0000
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_dd_italian.txt
+Cohen's d: 1.7005, p-value: 0.0000
+-----------------------------------------------
+

WEAT/weat_output_3.txt

+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_turkish_w2vformat.txt
+Cohen's d: 1.5315, p-value: 0.0003
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_turkish_w2vformat.txt
+Cohen's d: 1.1688, p-value: 0.0115
+-----------------------------------------------
+
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_polish_w2vformat.txt
+Cohen's d: 1.7833, p-value: 0.0000
+-----------------------------------------------
+
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 241
+    print("-----------------------------------------------)
+                                                          ^
+SyntaxError: EOL while scanning string literal
+srun: error: node37: task 0: Exited with exit code 1
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 241
+    print("-----------------------------------------------)
+                                                          ^
+SyntaxError: EOL while scanning string literal
+srun: error: node37: task 0: Exited with exit code 1
+  File "/home/students/reichelt/ba/bias-mitigation-ba/WEAT/weat_experiments.py", line 241
+    print("-----------------------------------------------)
+                                                          ^
+SyntaxError: EOL while scanning string literal
+srun: error: node37: task 0: Exited with exit code 1

WEAT/weat_output_4.txt

+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: italian test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_polish_w2vformat.txt
+Cohen's d: 1.2579, p-value: 0.0046
+-----------------------------------------------
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: turkish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_italian_w2vformat.txt
+Cohen's d: 1.7934, p-value: 0.0000
+-----------------------------------------------
+Loading seed words...
+Loading model...
+Retrieving embeddings...
+Calculating WEAT...
+WEAT scores for: polish test, vectors from /home/students/reichelt/ba/bias-mitigation-ba/data/embeddings/glove/dd-glove/glove_hard_debiased_italian_w2vformat.txt
+Cohen's d: 1.6358, p-value: 0.0000
+-----------------------------------------------