Delete unnecessary files

639a1aad · finn · a02d44bb · a02d44bb · a02d44bb · a02d44bb
Commit 639a1aad authored 1 year ago by finn
--- a/old/README.md
+++ b/old/README.md
--- a/old/example_chatgpt_annotation.txt
+++ b/old/example_chatgpt_annotation.txt
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots: None
-
-System: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots: None
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots:
-    - SlotType1: hotel-area
-    - SlotValue1: south
-
-System: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots:
-    - SlotType1: hotel-internet
-    - SlotValue1: yes
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots:
-    - SlotType1: hotel-parking
-    - SlotValue1: yes
-
-System: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots: None
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots:
-    - SlotType1: hotel-name
-    - SlotValue1: Aylesbray Lodge Guest House
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots:
-    - SlotType1: hotel-area
-    - SlotValue1: south
-    - SlotType2: hotel-bookday
-    - SlotValue2: Tuesday
-    - SlotType3: hotel-bookpeople
-    - SlotValue3: 4
-    - SlotType4: hotel-bookstay
-    - SlotValue4: 4
-    - SlotType5: hotel-internet
-    - SlotValue5: yes
-    - SlotType6: hotel-parking
-    - SlotValue6: yes
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots: None
-
-System: **BookingConfirmationCancellation | Slot:**
-
- Intent: BookingConfirmationCancellation
- Slots:
-    - SlotType1: hotel-name
-    - SlotValue1: Aylesbray Lodge Guest House
-    - SlotType2: hotel-bookday
-    - SlotValue2: Tuesday
-    - SlotType3: hotel-bookpeople
-    - SlotValue3: 4
-    - SlotType4: hotel-bookstay
-    - SlotValue4: 4
-    - SlotType5: hotel-area
-    - SlotValue5: south
-    - SlotType6: hotel-internet
-    - SlotValue6: yes
-    - SlotType7: hotel-parking
-    - SlotValue7: yes
-
-User: **Inquiry | Slot:**
-
- Intent: Inquiry
- Slots: None
-
-System: **Goodbye | Slot:**
-
- Intent: Goodbye
- Slots: None
\ No newline at end of file
--- a/old/example_chatgpt_dialogue.txt
+++ b/old/example_chatgpt_dialogue.txt
-User: Hello, I'd like to find a hotel in the city center with internet and parking.
-
-System: Thank you for using our service! Could you please specify the city or area you are looking for in the city center?
-
-User: I prefer the south area.
-
-System: Great choice! Do you require internet access during your stay?
-
-User: Yes, I need internet.
-
-System: Perfect! And would you also need parking facilities at the hotel?
-
-User: Yes, parking is essential.
-
-System: Thank you for providing the details. I have found a suitable hotel for you in the south area with internet and parking. It's the "Aylesbray Lodge Guest House." How many people will be staying, and for how many nights?
-
-User: There will be four of us staying for four nights.
-
-System: Thank you for the information. Your reservation for "Aylesbray Lodge Guest House" for four people, staying for four nights in the south area with internet and parking on Tuesday, is confirmed. Is there anything else I can assist you with?
-
-User: No, that's all. Thank you for your help!
-
-System: You're welcome! If you have any more questions in the future, feel free to ask. Have a great stay at Aylesbray Lodge Guest House! Goodbye!
\ No newline at end of file
--- a/old/example_triplets.txt
+++ b/old/example_triplets.txt
-(Service: hotel, Intent: find_hotel, Slot: hotel-area -> centre)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-internet -> yes)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-parking -> yes)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-area -> south)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-internet -> yes)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-parking -> yes)
-
-(Service: restaurant, Intent: find_restaurant)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-area -> south)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-internet -> yes)
-
-(Service: hotel, Intent: find_hotel, Slot: hotel-parking -> yes)
-
-(Service: restaurant, Intent: find_restaurant)
-
-(Service: hotel, Intent: Unknown Intent, Slot: hotel-name -> aylesbray lodge guest house)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-area -> south)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookday -> tuesday)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookpeople -> 4)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookstay -> 4)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-internet -> yes)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-parking -> yes)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-area -> south)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookday -> tuesday)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookpeople -> 4)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-bookstay -> 3)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-internet -> yes)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-name -> aylesbray lodge guest house)
-
-(Service: hotel, Intent: book_hotel, Slot: hotel-parking -> yes)
\ No newline at end of file
--- a/old/extract_domain_knowledge_txt.py
+++ b/old/extract_domain_knowledge_txt.py
-import json
-
-
-def extract_active_frames_from_dialogue(dialogue):
-    """Extracts frames with active intents from a single dialogue."""
-    # Extract frames where active_intent is not NONE
-    active_frames = [frame for turn in dialogue['turns'] for frame in turn.get('frames', []) if
-                     frame.get('state', {}).get('active_intent') != "NONE"]
-    return active_frames
-
-
-def format_frame(frame):
-    # Extract primary information
-    service = frame.get('service', 'Unknown Service')
-    active_intent = frame.get('state', {}).get('active_intent', 'Unknown Intent')
-
-    # Start with the service and intent
-    formatted_info = f"Service: {service}\nIntent: {active_intent}\n"
-
-    # Extract slots and their values
-    for slot in frame.get('slots', []):
-        slot_name = slot.get('slot', 'Unknown Slot')
-        value = slot.get('value', 'Unknown Value')
-        start = slot.get('start', '??')
-        end = slot.get('exclusive_end', '??')
-        formatted_info += f"    Slot: {slot_name} -> {value} (Position: {start}-{end})\n"
-
-    # Extract slot values from state
-    slot_values = frame.get('state', {}).get('slot_values', {})
-    if slot_values:
-        formatted_info += "    Slot Values:\n"
-        for key, values in slot_values.items():
-            formatted_info += f"        {key} -> {', '.join(values)}\n"
-
-    return formatted_info
-
-
-# Input the filename from the user
-filename = "dialogues.json"
-
-# Load the data from the file
-with open(filename, 'r') as file:
-    data = json.load(file)
-
-# Process and save each dialogue's frames separately
-for i, dialogue in enumerate(data):
-    extracted_frames = extract_active_frames_from_dialogue(dialogue)
-
-    if extracted_frames:  # Only save if there are active frames
-        formatted_text = ""
-        for frame in extracted_frames:
-            formatted_text += format_frame(frame)
-            formatted_text += '-' * 50 + '\n'  # Separate frames
-
-        output_filename = f"formatted_frames_dialogue_{i + 1}.txt"
-
-        # Save the formatted text to a file
-        with open(output_filename, 'w') as outfile:
-            outfile.write(formatted_text)
-
-        print(f"Saved formatted frames for dialogue {i + 1} to {output_filename}")
--- a/old/generate_dialogues_all.ipynb
+++ b/old/generate_dialogues_all.ipynb
--- a/old/llama_prompt_testing.ipynb
+++ b/old/llama_prompt_testing.ipynb
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "6de967f9-192f-40ee-b857-82e7e83a216d",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d4863b219fbe410c9fd9115328810cfe",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "from transformers import AutoTokenizer\n",
-    "\n",
-    "import transformers\n",
-    "import torch\n",
-    "\n",
-    "\n",
-    "model = \"meta-llama/Llama-2-13b-chat-hf\"\n",
-    "\n",
-    "tokenizer = AutoTokenizer.from_pretrained(model)\n",
-    "pipeline = transformers.pipeline(\n",
-    "    \"text-generation\",\n",
-    "    model=model,\n",
-    "    torch_dtype=torch.float16,\n",
-    "    device_map=\"auto\",\n",
-    "    temperature=0.25\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "269aea0f-1765-4fe5-bcd5-03e9f8f2d2e0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sequences = pipeline(\n",
-    "    \"\"\"[INST] <<SYS>>\\n\n",
-    "Simulate to be a system and respond to a user. Only generate system and user dialogue, without any action descriptions.\n",
-    "The user requests one or more specific acts of service as expected by an online service system. The system asks the user for necessary details, verifies the applicability of the user’s requests to the Domain Knowledge and then after potential adjustments, provides the service. Every user message should be followed by a system message. Always remain courteous and ensure to bid farewell. All statements made by the system should adhere to the Domain Knowledge provided.\n",
-    "Generate the complete dialogue between the system and the user.\n",
-    "Domain Knowledge:\n",
-    "{\n",
-    "        \"Services Involved\": [\n",
-    "            \"train\",\n",
-    "            \"attraction\"\n",
-    "        ],\n",
-    "        \"Service Details\": [\n",
-    "            {\n",
-    "                \"Service Type\": \"train\",\n",
-    "                \"Actions\": [\n",
-    "                    \"NONE\",\n",
-    "                    \"find_train\"\n",
-    "                ],\n",
-    "                \"Preferences\": {\n",
-    "                    \"train-departure\": [\n",
-    "                        \"cambridge\"\n",
-    "                    ],\n",
-    "                    \"train-destination\": [\n",
-    "                        \"peterborough\"\n",
-    "                    ],\n",
-    "                    \"train-arriveby\": [\n",
-    "                        \"14:45\"\n",
-    "                    ],\n",
-    "                    \"train-day\": [\n",
-    "                        \"friday\"\n",
-    "                    ]\n",
-    "                }\n",
-    "            },\n",
-    "            {\n",
-    "                \"Service Type\": \"attraction\",\n",
-    "                \"Actions\": [\n",
-    "                    \"find_attraction\",\n",
-    "                    \"NONE\"\n",
-    "                ],\n",
-    "                \"Preferences\": {\n",
-    "                    \"attraction-area\": [\n",
-    "                        \"centre\"\n",
-    "                    ],\n",
-    "                    \"attraction-type\": [\n",
-    "                        \"museum\"\n",
-    "                    ]\n",
-    "                }\n",
-    "            }\n",
-    "        ]\n",
-    "    },\n",
-    "\\n<</SYS>>\\n\\n [/INST]\n",
-    "\"\"\",\n",
-    "    do_sample=True,\n",
-    "    top_k=10,\n",
-    "    num_return_sequences=1,\n",
-    "    eos_token_id=tokenizer.eos_token_id,\n",
-    "    max_length=6000,\n",
-    ")\n",
-    "for seq in sequences:\n",
-    "    print(f\"Result: {seq['generated_text']}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "03d44752-1432-4cab-b517-35f354ff5a0b",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.7"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
-%% Cell type:code id:6de967f9-192f-40ee-b857-82e7e83a216d tags:
-
-``` python
-from transformers import AutoTokenizer
-
-import transformers
-import torch
-
-
-model = "meta-llama/Llama-2-13b-chat-hf"
-
-tokenizer = AutoTokenizer.from_pretrained(model)
-pipeline = transformers.pipeline(
-    "text-generation",
-    model=model,
-    torch_dtype=torch.float16,
-    device_map="auto",
-    temperature=0.25
-)
-```
-
-%% Output
-
-
-%% Cell type:code id:269aea0f-1765-4fe5-bcd5-03e9f8f2d2e0 tags:
-
-``` python
-sequences = pipeline(
-    """[INST] <<SYS>>\n
-Simulate to be a system and respond to a user. Only generate system and user dialogue, without any action descriptions.
-The user requests one or more specific acts of service as expected by an online service system. The system asks the user for necessary details, verifies the applicability of the user’s requests to the Domain Knowledge and then after potential adjustments, provides the service. Every user message should be followed by a system message. Always remain courteous and ensure to bid farewell. All statements made by the system should adhere to the Domain Knowledge provided.
-Generate the complete dialogue between the system and the user.
-Domain Knowledge:
-{
-        "Services Involved": [
-            "train",
-            "attraction"
-        ],
-        "Service Details": [
-            {
-                "Service Type": "train",
-                "Actions": [
-                    "NONE",
-                    "find_train"
-                ],
-                "Preferences": {
-                    "train-departure": [
-                        "cambridge"
-                    ],
-                    "train-destination": [
-                        "peterborough"
-                    ],
-                    "train-arriveby": [
-                        "14:45"
-                    ],
-                    "train-day": [
-                        "friday"
-                    ]
-                }
-            },
-            {
-                "Service Type": "attraction",
-                "Actions": [
-                    "find_attraction",
-                    "NONE"
-                ],
-                "Preferences": {
-                    "attraction-area": [
-                        "centre"
-                    ],
-                    "attraction-type": [
-                        "museum"
-                    ]
-                }
-            }
-        ]
-    },
-\n<</SYS>>\n\n [/INST]
-""",
-    do_sample=True,
-    top_k=10,
-    num_return_sequences=1,
-    eos_token_id=tokenizer.eos_token_id,
-    max_length=6000,
-)
-for seq in sequences:
-    print(f"Result: {seq['generated_text']}")
-```
-
-%% Cell type:code id:03d44752-1432-4cab-b517-35f354ff5a0b tags:
-
-``` python
-```
--- a/old/preprocess_data_for_metrics/grade/csv_dialogue_to_2_files.py
+++ b/old/preprocess_data_for_metrics/grade/csv_dialogue_to_2_files.py
-import pandas as pd
-import ast
-
-# Load the CSV file into a DataFrame
-df = pd.read_csv('data/own_data/multiwoz_2.2_processed.csv')  
-
-file_1 = 'human_ctx.txt'  
-file_2 = 'human_hyp.txt' 
-file_3 = 'dialogues_text.txt'
-
-with open(file_1, 'w') as f1, open(file_2, 'w') as f2, open(file_3, 'w') as f3:
-    for i, row in df['utterances_separate'].items():
-        utterances_list = ast.literal_eval(row)
-        if (len(utterances_list)%3) !=0: utterances_list_new = utterances_list[:-(len(utterances_list)%3)]
-        else: utterances_list_new = utterances_list
-        first = 1
-        for index, utterance in enumerate(utterances_list_new):
-            f3.write(utterance + ' __eou__ ')
-            if ((index+1) % 3) != 0:
-                if first:
-                    f1.write(f"{utterance}|||")
-                    first += -1
-                else:
-                    f1.write(f"{utterance}\n")
-                    first += 1
-            elif ((index+1) % 3) == 0:
-                f2.write(f"{utterance}\n")
-        #if i == 50: break
\ No newline at end of file
--- a/old/preprocess_data_for_metrics/multiwoz_2.2_processed.csv
+++ b/old/preprocess_data_for_metrics/multiwoz_2.2_processed.csv