Commit 5f5b8342 authored by Aileen Reichelt's avatar Aileen Reichelt
Browse files

Implement basis for sentence length analysis

parent 74f40e6f
Loading
Loading
Loading
Loading

sentence_length.py

0 → 100644
+17 −0
Changes for sentence_length.py: 17 added lines, 0 removed lines.
Original line number Diff line number Diff line
from SBERT_Model import SBERT_Model
from datasets import load_dataset
import pandas

dataset = load_dataset('csv', data_files='data/stsbenchmark/sts-test.csv', split='test')
dataset.set_format(type='pandas')
print(dataset.head())

short_sentences = pandas.DataFrame
average_length_sentences = pandas.DataFrame
long_sentences = pandas.DataFrame

datasets = [short_sentences, average_length_sentences, long_sentences]

for d in datasets:
    ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', d)
    ft_model.print_statistics()