Remove old files

fanfic_preprocessing.py

@@ -157,7 +157,7 @@ def run_functions(file_paths):
 
 if __name__ == "__main__":
 
-    clean_fanfic_dataset("cosmere/data/fanfics/cosmere_fanfics.csv")
+    #clean_fanfic_dataset("cosmere/data/fanfics/cosmere_fanfics.csv")
 
     #clean_fanfic_dataset("grishaverse/data/fanfics/grisha_fanfics.csv")
 
@@ -165,7 +165,16 @@ if __name__ == "__main__":
     
     #file_paths = ["cosmere/data/fanfics/cosmere_fanfics.csv"]
     #file_paths = ["grishaverse/data/fanfics/grisha_fanfics_new.csv"]
-    #run_functions(file_paths=file_paths)
+
+    file_path_list = ['call_me_by_your_name/data', 'cosmere/data', 'divergent/data', 'grishaverse/data', 'maze_runner/data', 'murderbot/data', 'percy/data', 'red_white_royal_blue/data', 'school_for_good_and_evil/data', 'simonverse/data', 'song_of_achilles/data', 'throne_of_glass/data',]
+    file_paths_lists = [os.listdir(file_path) for file_path in file_path_list]
+    file_paths = []
+    for file_path_list in file_paths_lists:
+        file_paths.extend(file_paths)
+    print(file_paths)
+    run_functions(file_paths=file_paths)
+
+
     """
     cosmere = pd.read_csv("cosmere/data/fanfics/cosmere_fanfics_new.csv")
     separate_fanfics_by_good_medium_bad(cosmere, "cosmere", 36, 73) # I select the thresholds to have a 50% 25% 25% split