diff --git a/count_name_occurrences.py b/count_name_occurrences.py index 73bcf9ec8c32a897b72d64ea7dceff309ecc73cb..fc1b8f4b5a5616271793c95c0a05f0958c14fc21 100644 --- a/count_name_occurrences.py +++ b/count_name_occurrences.py @@ -8,7 +8,7 @@ df["occurrences_in_wikipedia"] = 0 CHUNK_SIZE = 1024 * 1024 # 1 MB -with open("../data/wikipedia/wikipedia_corpus.txt", "rb", encoding="utf-8") as f: +with open("../data/wikipedia/wikipedia_corpus.txt", "rb") as f: CHUNK_NO = 1 while True: print(f"reading Wikipedia data: approx. {CHUNK_NO} MB/8100 MB", end="\r") @@ -18,13 +18,13 @@ with open("../data/wikipedia/wikipedia_corpus.txt", "rb", encoding="utf-8") as f break for index, row in df.iterrows(): - name = row['name'] + name = row['name'].encode() count = wikipedia_text.count(name) df.at[index, "occurrences_in_wikipedia"] += count CHUNK_NO += 1 -df.to_csv("./data/names_nationality_wikipedia.csv", index=False) +df.to_csv("../data/names_nationality_wikipedia.csv", index=False) avg_occurrences = df.groupby("nationality")["occurrences_in_wikipedia"].mean() for nationality, avg_count in avg_occurrences.items():