diff --git a/clean_stylometry.py b/clean_stylometry.py
index 592f58dfaf95c02219e3005b00aa81d25ba10046..32d8a7d70fdb55a2effd571d9ca1d7c8682eb79c 100644
--- a/clean_stylometry.py
+++ b/clean_stylometry.py
@@ -255,26 +255,22 @@ def calculate_sent_len_dist(text):
return sent_len_dist, sent_len_dist_short, standard_deviation_sent, mean_sent
-#f"throne_of_glass/data/canon_works"
-def extract_info_from_directory_path(directory_path):
- #for txt_fic in os.listdir(directory_path):
- works = os.listdir(directory_path)
+def extract_info_from_dir_path(dir_path):
pattern = r"^[a-zA-Z_]+(?=/)" # get series from directory path
- match = re.search(pattern, directory_path)
+ match = re.search(pattern, dir_path)
if match:
- series = match.group(0)
- for work in works:
- with open(f"{directory_path}"+f"/{work}", "r") as f:
- f = f.read()
- std_dev_tk, mean_tk, ttr = mendenhall_curve(f, f"Mendenhall Curve for the {series.replace('_' , ' ').title()} {work[:-4].replace('_' , ' ').title()}", f"{series}/freq_distribution/{work[:-4]}_token_len.png")
- mean_tokens.append(mean_tk)
+ name_of_universe = match.group(0)
+ print(name_of_universe)
+ name_of_work = name_of_universe.replace("_", " ").title()
+ return name_of_work, name_of_universe
class StylometryMetrics:
- def __init__(self, directory_path, name_of_work, name_of_universe, quality="", fanfiction=True):
+ def __init__(self, directory_path, quality="", fanfiction=False):
self.text = read_works_into_string(directory_path)
self.clean_tokens = tokenize_and_clean_text(self.text)
+ name_of_work, name_of_universe = extract_info_from_dir_path(dir_path=directory_path)
self.name = name_of_work
self.fanfiction = fanfiction
self.quality = quality # good medium bad
@@ -425,6 +421,12 @@ def create_dataframe_with_overview_info():
if __name__ == "__main__":
+ # canon calculations
+
+ # fanfic calculations
+
+
+
#run_functions("grishaverse/data/split_txt_fanfics")
#run_functions("throne_of_glass/data/split_txt_fanfics")
#data_overview.to_csv(f"data_overview/data_overview.csv")
@@ -444,4 +446,4 @@ if __name__ == "__main__":
#GrishaverseCanon.plot_md_freq("grishaverse/plots/canon/md_freq.png")
-
\ No newline at end of file
+
diff --git a/grisha_fanfics.csv b/grisha_fanfics.csv
index e0ac15178c02cc3f2f374fa59a93bcf8b5d920d9..3cbc3984ad548a119baea7e5f3a62ba22f56355c 100644
Binary files a/grisha_fanfics.csv and b/grisha_fanfics.csv differ
diff --git a/mazerunner_fanfics.csv b/mazerunner_fanfics.csv
index d6d850f89d3591de5e36131f5b595992c7b41895..efc7f3d7e8b88bfd38aff7f944c68a8f939a214e 100644
Binary files a/mazerunner_fanfics.csv and b/mazerunner_fanfics.csv differ
diff --git a/percy_fanfics.csv b/percy_fanfics.csv
index 388b802f3f0cc0920b8027984449d2a129d55631..6f3374476e9aef8fe202c1eadd907abcbbebe23e 100644
Binary files a/percy_fanfics.csv and b/percy_fanfics.csv differ
diff --git a/stylometry_code.py b/stylometry_code.py
deleted file mode 100644
index cad4c6d207802230c6413aed7005e0bd2f41f8bd..0000000000000000000000000000000000000000
--- a/stylometry_code.py
+++ /dev/null
@@ -1,489 +0,0 @@
-import seaborn as sns
-import matplotlib.pyplot as plt
-from cycler import cycler
-import os
-from nltk.tokenize import word_tokenize
-from nltk.probability import FreqDist
-from nltk.tokenize import sent_tokenize
-from nltk.tag import pos_tag
-import pandas as pd
-import statistics
-import re
-
-
-# you'll have to also download "punkt" from nltk
-
-# create function for bar (value) labels
-def addlabels(x,y):
- for i in range(len(x)):
- plt.text(i, y[i], y[i], ha = "center")
-
-
-# function compiling the works given into a single string. Input required:
-# general path of the files as string, for example: "/throne_of_glass/data/canon_works/"
-# specific names of the works as a list of strings, for example "throne_of_glass_1.txt"
-
-# /throne_of_glass/data/canon_works/
-def read_works_into_string(directory_path):
- strings = []
- works = os.listdir(directory_path)
- for work in works:
- with open(f"{directory_path}"+f"/{work}", "r") as f:
- strings.append(f.read())
- return "\n".join(strings)
-
-
-# by subdiving the text into segments of 1000, it calculates the type token ratio for each segment and then averages over them
-# this ensures a comparability of the type token ratios for varying text sizes
-def standardised_type_token_ratio(tokens):
- ttrs = []
- segment_tokens = []
- segment = 0
- for token in tokens:
- if segment < 1000:
- segment_tokens.append(token)
- segment += 1
- elif segment == 1000:
- types = set(segment_tokens)
- ttr = len(types)/len(segment_tokens)
- ttrs.append(ttr)
- segment_tokens =[]
- segment = 0
- if len(ttrs) <= 1:
- types = set(tokens)
- std_ttr = len(types)/len(tokens)
- print("Warning: Text was too short for segmentation!")
- else:
- std_ttr = statistics.mean(ttrs)
- return std_ttr
-
-
-def tokenize_and_clean_text(text):
-
- tokens = word_tokenize(text)
- cleaned_tokens = ([token for token in tokens if any(c.isalpha() for c in token)])
- short_clean_tokens = [] # when looking at the results, there were some strange token lengths, because somewhere in the data conversion hyphens
- # had been added in the wrong places. I had the tokens with very large lengths printed and they had this format, e.g. "everywhere—assassin"
- # and where counted, in this instance as 19 characters long but up to 45 characters long: "walking-as-fast-as-they-could-without-running"
-
- for token in cleaned_tokens:
- dehyphenated_token = []
- letter_present = 0
- dehyphenated = 0
- second_word_in_compound = 0
- for c in token:
- if c.isalpha() == True:
- dehyphenated_token.append(c)
- letter_present = 1
- if dehyphenated == 1:
- second_word_in_compound = 1
- elif c.isalpha() == False and letter_present == 1: #here I am eliminating both dashes and hyphens,
- #bc it skews the word metric if red-blue is counted as a 9 character token, boosting the count of
- # high-character tokens significantly. all texts will be preprocessed the same way, so it shouldn't make a difference,
- # relatively speaking
- dehyphenated_token_joined = ''.join(map(str, dehyphenated_token))
- #print(dehyphenated_token_joined)
- short_clean_tokens.append(dehyphenated_token_joined)
- dehyphenated_token = []
- letter_present = 0
- dehyphenated = 1
- second_word_in_compound = 0
- if letter_present == 1 and dehyphenated == 0:
- short_clean_tokens.append(token) #catching the tokens that didn't have any special characters; but not the dehyphenated ones twice
- elif letter_present == 1 and dehyphenated == 1 and second_word_in_compound == 1:
- short_clean_tokens.append(''.join(map(str, dehyphenated_token)))
- return short_clean_tokens
-
-
-
-# this function takes a corpus as its input and gives a Mendenhall curve, i.e. a frequency distribution of tokens as its output
-# precise input: corpus = string ;
-# curve_title = string, the title of the plot that will be produced, e.g., "Mendenhall Curve for Throne of Glass Series"
-# plot_destination = string, the (relative) path, including the file name and .png tag of the plot produced, e.g. f"throne_of_glass/freq_distribution/all_canon_token_len.png"
-
-
-def mendenhall_curve(corpus, curve_title, plot_destination):
-
- short_clean_tokens = tokenize_and_clean_text(corpus)
-
- # create the distribution of token lengths / Mendenhall curve
-
- token_lengths = [len(token) for token in short_clean_tokens]
-
- # Calculate the trimmed token length (with 5% trimming) We need to remove the outliers, bc even despite preprocessing,
- # there still are some very wrong lengths, which entirely skews the metrics and also ruins our p-values later on
- trim_percent = 0.005
- trim_len = int(len(token_lengths) * trim_percent / 2)
- token_lengths = sorted(token_lengths)[trim_len:-trim_len]
-
-
- token_length_distribution = FreqDist(token_lengths).most_common(15)
-
- # convert to FreqDist object to a pandas series for easier processing
- token_len_dist_panda = pd.Series(dict(token_length_distribution))
-
- # sort, normalise and round the panda series
-
- new_token_len_dist = token_len_dist_panda.sort_index()
-
- for i in range(0, len(new_token_len_dist.index)):
- #for index in new_token_len_dist.index:
- new_token_len_dist.iat[i] = round(new_token_len_dist.iat[i]/len(short_clean_tokens), 3) #index-1 bc the index starts counting from zero, the word lengths not
- #if float(new_token_len_dist.iat[i]) == 0.00:
- # new_token_len_dist.drop(index=i) # here it is used as the label, so we want the index, not index -1; bad work-around, I'm sorry
-
-
- # plot using matplotlib and seaborn
-
- # set figure, ax into variables
- fig, ax = plt.subplots(figsize=(10,10))
-
- # call function for bar (value) labels
- addlabels(x=new_token_len_dist.index, y=new_token_len_dist.values)
-
- plt.title(curve_title)
- ax.set_xlabel("Word Length")
- ax.set_ylabel("Percentage of Occurence")
-
- sns.barplot(x=new_token_len_dist.index, y=new_token_len_dist.values, ax=ax, palette="flare")
- #plt.xticks(rotation=30) !!! very useful for words
- #plt.get_figure()
- plt.savefig(plot_destination)
- #print(new_token_len_dist.tabulate())
- #token_length_freq_dist_plot = token_length_distribution.plot(title=curve_title, percents=True)
-
- #fig_freq_dist = token_length_freq_dist_plot.get_figure()
- #fig_freq_dist.savefig(plot_destination)
-
- # calculate the standard deviation, mean, token/type ratio
- standard_deviation = statistics.stdev(token_lengths)
- mean = statistics.mean(token_lengths)
-
- type_token_ratio = standardised_type_token_ratio(short_clean_tokens)
-
- return standard_deviation, mean, type_token_ratio
-
-
-def sentence_metrics(corpus, curve_title, series, canon_or_fanfic):
-
- sents = sent_tokenize(corpus)
- sent_lens = []
- for sent in sents:
- short_clean_tokens = tokenize_and_clean_text(sent)
- sent_lens.append(len(short_clean_tokens))
- #if len(short_clean_tokens)>= 90:
- #print(f"This sentence: \n {sent} \n is this long: {len(short_clean_tokens)}")
-
- # Calculate the trimmed mean sentence length (with 5% trimming) We need to remove the outliers, bc even despite preprocessing,
- # there still are some sentences that are 1200 tokens long, which entirely skews the metrics and also ruins our p-values later on
- trim_percent = 0.05
- trim_len = int(len(sent_lens) * trim_percent / 2)
- sent_lens = sorted(sent_lens)[trim_len:-trim_len]
-
-
- sent_len_dist = FreqDist(sent_lens)
- #print(sent_len_dist)
-
- # convert to FreqDist object to a pandas series for easier processing
- sent_len_dist_panda = pd.Series(dict(sent_len_dist))
-
- # sort, normalise and round the panda series
-
- new_sent_len_dist = sent_len_dist_panda.sort_index()
- #print(new_sent_len_dist)
-
- for i in range(0, len(new_sent_len_dist.index)):
- #for index in new_token_len_dist.index:
- new_sent_len_dist.iat[i] = round(new_sent_len_dist.iat[i]/len(sent_lens), 2) #index-1 bc the index starts counting from zero, the word lengths not
-
- #print(new_sent_len_dist)
- # plot using matplotlib and seaborn
-
- # set figure, ax into variables
- fig, ax = plt.subplots(figsize=(10,10))
-
- # call function for bar (value) labels
- #addlabels(x=new_sent_len_dist.index, y=new_sent_len_dist.values)
-
- plt.title(curve_title)
- ax.set_xlabel("Sentence Length")
- ax.set_ylabel("Percentage of Occurence")
-
-
- sns.lineplot(x=new_sent_len_dist.index, y=new_sent_len_dist.values, ax=ax, palette="crest")
- #plt.xticks(rotation=30) !!! very useful for words
- plt.savefig(f"{series}/freq_distribution/{canon_or_fanfic}_sent_len_long.png") # "throne_of_glass/freq_distribution/all_canon_sent_len.png"
-
- # plot the 40 most frequent sentence lenghts as a barplot for a more detailed insight
- sent_len_dist_short = FreqDist(sent_lens).most_common(25)
-
- # convert to FreqDist object to a pandas series for easier processing
- sent_len_dist_short_panda = pd.Series(dict(sent_len_dist_short))
-
- # sort, normalise and round the panda series
-
- new_sent_len_dist_short = sent_len_dist_short_panda.sort_index()
- #print(new_sent_len_dist)
-
- for i in range(0, len(new_sent_len_dist_short.index)):
- #for index in new_token_len_dist.index:
- new_sent_len_dist_short.iat[i] = round(new_sent_len_dist_short.iat[i]/len(sent_lens), 2) #index-1 bc the index starts counting from zero, the word lengths not
-
- # set figure, ax into variables
- fig, ax = plt.subplots(figsize=(10,10))
-
- # call function for bar (value) labels
- addlabels(x=new_sent_len_dist_short.index, y=new_sent_len_dist_short.values)
-
- plt.title(curve_title)
- ax.set_xlabel("Sentence Length")
- ax.set_ylabel("Percentage of Occurence")
-
- sns.barplot(x=new_sent_len_dist_short.index, y=new_sent_len_dist_short.values, ax=ax, palette="YlGnBu")
- #plt.xticks(rotation=30) !!! very useful for words
- plt.savefig(f"{series}/freq_distribution/{canon_or_fanfic}_sent_len_short.png") # "throne_of_glass/freq_distribution/all_canon_sent_len.png"
-
- # calculate the standard deviation, mean, token/type ratio
- standard_deviation_sent = statistics.stdev(sent_lens)
- mean_sent = statistics.mean(sent_lens)
-
- return standard_deviation_sent, mean_sent
-
-
-# overall pos_tag frequency distribution
-# pos_tag ngrams; (maybe exclude stopwords?)
-# tag collocates for specific tags --> adjectives most frequently with nouns
-# most frequent words
-# most frequent words for specific tags --> punctuation;
-# most frequent adjectives
-
-def pos_tag_frequencies(corpus, series, canon_or_fanfic):
- #nltk.pos_tag(text) --> [('And', 'CC'), ('now', 'RB'), ('for', 'IN'), ('something', 'NN'),
- #('completely', 'RB'), ('different', 'JJ')]
- tokens = word_tokenize(corpus)
- """
- short_tokens = []
- for token in tokens:
- dehyphenated_token = []
- letter_present = 0
- dehyphenated = 0
- second_word_in_compound = 0
- for c in token:
- if c.isalpha() == True:
- dehyphenated_token.append(c)
- letter_present = 1
- if dehyphenated == 1:
- second_word_in_compound = 1
- elif c.isalpha() == False and letter_present == 1: #here I am eliminating both dashes and hyphens,
- #bc it skews the word metric if red-blue is counted as a 9 character token, boosting the count of
- # high-character tokens significantly. all texts will be preprocessed the same way, so it shouldn't make a difference,
- # relatively speaking
- dehyphenated_token_joined = ''.join(map(str, dehyphenated_token))
- #print(dehyphenated_token_joined)
- short_tokens.append(dehyphenated_token_joined)
- short_tokens.append(c) #append the hyphen/ other punctuation --> we're also interested in that
- dehyphenated_token = []
- letter_present = 0
- dehyphenated = 1
- second_word_in_compound = 0
- if letter_present == 1 and dehyphenated == 0:
- short_tokens.append(token) #catching the tokens that didn't have any special characters; but not the dehyphenated ones twice
- elif letter_present == 1 and dehyphenated == 1 and second_word_in_compound == 1:
- short_tokens.append(''.join(map(str, dehyphenated_token)))
- """
- tag_token_tuples = pos_tag(tokens)
- punctuation_regex = r"[^\w\s]+"
- summarised_tags = []
- punctuation_tags = []
- index = 0
- for token, tag in tag_token_tuples:
- if re.match(punctuation_regex, token):
- summarised_tags.append("punctuation")
- if re.match(r"[\"\'“â€â€™â€˜]+", token):
- punctuation_tags.append("quotation_marks")
- elif re.match(r"[,;:.?!-]+", token):
- try:
- punctuation_tags.append("ellipsis" if token == "." and tag_token_tuples[index+1][1] == "." and tag_token_tuples[index+2][1] == "." else "full_stop" if token == "." else "question_mark" if token == "?" else "exclamation_mark" if token == "!" else "comma" if token == "," else "semicolon" if token == ";" else "dash" if token == "-" else "other_punct")
- except:
- punctuation_tags.append("full_stop" if token == "." else "question_mark" if token == "?" else "exclamation_mark" if token == "!" else "comma" if token == "," else "semicolon" if token == ";" else "dash" if token == "-" else "other_punct")
-
- else:
- if tag in ["MD", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ"]:
- summarised_tags.append("verb")
- elif tag in ["JJ", "JJR", "JJS"]:
- summarised_tags.append("adjective")
- elif tag in ["RB", "RBR", "RBS", "WRB"]:
- summarised_tags.append("adverb")
- elif tag in ["PRP", "PRP$", "WP", "WP$"]:
- summarised_tags.append("pronoun")
- elif tag in ["NNP", "NNPS"]:
- summarised_tags.append("proper_noun")
- elif tag in ["NN", "NNS"]:
- summarised_tags.append("common_noun")
- elif tag in ["DT", "PDT", "WDT"]:
- summarised_tags.append("determiner")
- elif tag == "CC":
- summarised_tags.append("coordinating_conj")
- elif tag == "IN":
- summarised_tags.append("subordinating_conj")
- elif tag in ["$", "CD", "EX", "LS", "POS", "SYM", "TO", "UH", "RP", "FW"]:
- summarised_tags.append("other_tag")
- index += 1
-
-
- tag_freq_dist = FreqDist(summarised_tags)
- #print(tag_freq_dist)
-
- # convert FreqDist object to a pandas series for easier processing
- tag_freq_dist_panda = pd.Series(dict(tag_freq_dist))
- #print(tag_freq_dist_panda)
-
- # sort, normalise and round the panda series
-
- new_tag_freq_dist = tag_freq_dist_panda.sort_index()
- #print(new_sent_len_dist)
-
- for i in range(0, len(new_tag_freq_dist.index)):
- #for index in new_token_len_dist.index:
- new_tag_freq_dist.iat[i] = round(new_tag_freq_dist.iat[i]/len(tag_token_tuples), 2) #index-1 bc the index starts counting from zero, the word lengths not
-
- print(new_tag_freq_dist)
-
- # set figure, ax into variables
- fig, ax = plt.subplots(figsize=(10,10))
-
- # call function for bar (value) labels
- addlabels(x=new_tag_freq_dist.index, y=new_tag_freq_dist.values)
-
- plt.title(f"POS Tag Frequencies for the {series.replace('_' , ' ').title()} {canon_or_fanfic.replace('_' , ' ').title()}")
- ax.set_xlabel("POS Tags")
- ax.set_ylabel("Percentage of Occurence")
-
- sns.barplot(x=new_tag_freq_dist.index, y=new_tag_freq_dist.values, ax=ax, palette="RdPu")
- plt.xticks(rotation=30) # !!! very useful for words
- plt.savefig(f"{series}/freq_distribution/{canon_or_fanfic}_pos_tag_frequencies.png") # "throne_of_glass/freq_distribution/all_canon_sent_len.png"
-
-
- #punctuation frequency distribution
-
- punct_tag_freq_dist = FreqDist(punctuation_tags)
- #print(tag_freq_dist)
-
- # convert FreqDist object to a pandas series for easier processing
- punct_tag_freq_dist_panda = pd.Series(dict(punct_tag_freq_dist))
- #print(punct_tag_freq_dist_panda)
-
- # sort, normalise and round the panda series
-
- new_punct_tag_freq_dist = punct_tag_freq_dist_panda.sort_index()
- #print(new_sent_len_dist)
-
- for i in range(0, len(new_punct_tag_freq_dist.index)):
- #for index in new_token_len_dist.index:
- new_punct_tag_freq_dist.iat[i] = round(new_punct_tag_freq_dist.iat[i]/len(punctuation_tags), 3) #index-1 bc the index starts counting from zero, the word lengths not
-
- #print(new_punct_tag_freq_dist)
-
- # set figure, ax into variables
- fig, ax = plt.subplots(figsize=(10,10))
-
- # call function for bar (value) labels
- addlabels(x=new_punct_tag_freq_dist.index, y=new_punct_tag_freq_dist.values)
-
-
- plt.title(f"Punctuation Frequencies for the {series.replace('_' , ' ').title()} {canon_or_fanfic.replace('_' , ' ').title()}")
- ax.set_xlabel("Types of Punctuation")
- ax.set_ylabel("Percentage of Occurence")
-
- sns.barplot(x=new_punct_tag_freq_dist.index, y=new_punct_tag_freq_dist.values, ax=ax, palette="OrRd")
- plt.xticks(rotation=30) # !!! very useful for words
- plt.savefig(f"{series}/freq_distribution/{canon_or_fanfic}_punctuation_frequencies.png") # "throne_of_glass/freq_distribution/all_canon_sent_len.png"
-
-
-#create the Mendenhall Curve for the Throne of Glass Series
-#std_dev_tokens_tog_canon, mean_tokens_tog_canon, type_token_ratio_tog_canon = mendenhall_curve(read_works_into_string(f"throne_of_glass/data/canon_works"), "Mendenhall Curve for the Throne of Glass Series", f"throne_of_glass/freq_distribution/all_canon_token_len.png")
-
-#create the Mendenhall Curve for the Grishaverse Books
-#std_dev_tokens_grishaverse_canon, mean_tokens_grishaverse_canon, type_token_ratio_grishaverse_canon = mendenhall_curve(read_works_into_string(f"grishaverse/data/canon_works"), "Mendenhall Curve for the Grishaverse Books", f"grishaverse/freq_distribution/all_canon_token_len.png")
-
-
-# Mendenhall Curve Sentence Lengths for Throne of Glass Canon
-#std_dev_sent_tog_canon, mean_sent_tog_canon = sentence_metrics(read_works_into_string(f"throne_of_glass/data/canon_works"), "Mendenhall Curve for Sentence Lenghts for the Throne of Glass Series", "throne_of_glass", "canon")
-
-# Mendenhall Curve Sentence Lenghts for Grishavers Canon
-#std_dev_sent_grishaverse_canon, mean_sent_grishaverse_canon = sentence_metrics(read_works_into_string(f"grishaverse/data/canon_works"), "Mendenhall Curve for Sentence Lenghts for the Grishaverse Books", "grishaverse", "canon")
-
-# POS Tag frequencies for TOG
-#pos_tag_frequencies(read_works_into_string(f"throne_of_glass/data/canon_works"), "throne_of_glass", "canon")
-
-# POS Tag frequencies for Grishaverse
-#pos_tag_frequencies(read_works_into_string(f"grishaverse/data/canon_works"), "grishaverse", "canon")
-
-def run_functions(directory_path):
- """
- mean_tks = []
- idx = []
- std_dev_tks = []
- ttrs = []
- mean_sts= []
- std_dev_sts = []
-
- """
-
- #for txt_fic in os.listdir(directory_path):
- works = os.listdir(directory_path)
- pattern = r"^[a-zA-Z_]+(?=/)" # get series from directory path
- match = re.search(pattern, directory_path)
- if match:
- series = match.group(0)
- for work in works:
- with open(f"{directory_path}"+f"/{work}", "r") as f:
- f = f.read()
- std_dev_tk, mean_tk, ttr = mendenhall_curve(f, f"Mendenhall Curve for the {series.replace('_' , ' ').title()} {work[:-4].replace('_' , ' ').title()}", f"{series}/freq_distribution/{work[:-4]}_token_len.png")
- mean_tokens.append(mean_tk)
- std_dev_tokens.append(std_dev_tk)
- type_token_ratio.append(ttr)
- std_dev_st, mean_st = sentence_metrics(f, f"Mendenhall Curve for Sentence Lenghts for the {series.replace('_' , ' ').title()} {work[:-4].replace('_' , ' ').title()}", series, work[:-4])
- mean_sent.append(mean_st)
- std_dev_sents.append(std_dev_st)
- pos_tag_frequencies(f, series, work[:-4])
- index.append(f"{series}_{work[:-4]}")
-
-
-#grishaverse/data/split_txt_fanfics
-
-#create lists for each of the columns of the dataframe we'll create
-
-mean_tokens = [mean_tokens_tog_canon, mean_tokens_grishaverse_canon]
-std_dev_tokens = [std_dev_tokens_tog_canon, std_dev_tokens_grishaverse_canon]
-type_token_ratio = [type_token_ratio_tog_canon, type_token_ratio_grishaverse_canon]
-mean_sent = [mean_sent_tog_canon, mean_sent_grishaverse_canon]
-std_dev_sents = [std_dev_sent_tog_canon, std_dev_sent_grishaverse_canon]
-index = ["throne_of_glass_canon", "grishaverse_canon"]
-
-
-#run_functions("grishaverse/data/split_txt_fanfics")
-#run_functions("throne_of_glass/data/split_txt_fanfics")
-
-# create a dataframe to store all the overview statistics in
-# columns mean_tokens; std_dev_tokens; freq_token_len_1; ...; freq_token_len_15;
-# mean_sent; std_dev_sent; freq_sent_len ....
-# tag_frequencies
-# tag_ngram_frequencies
-# punctuation frequencies
-# token/type ratio
-
-data_overview = pd.DataFrame(
- {"mean_tokens":mean_tokens,
- "std_dev_tokens":std_dev_tokens,
- "type_token_ratio":type_token_ratio,
- "mean_sent":mean_sent,
- "std_dev_sent":std_dev_sents},
- index = index
-)
-
-if __name__ == "__main__":
-
- run_functions("grishaverse/data/split_txt_fanfics")
- run_functions("throne_of_glass/data/split_txt_fanfics")
- data_overview.to_csv(f"data_overview/data_overview.csv")