test_scanner.py

# -*- coding: utf-8 -*-

import allzweckmesser as azm


def test_tokenize():
    expected = ['clamavit', 'moriens', 'lingua', ':', "'", 'Corinna', ',',
                'vale', '!', "'"]
    tokens = azm.scanner.tokenize("clamavit moriens lingua: 'Corinna, vale!'")
    plain_tokens = [t.text for t in tokens]
    assert plain_tokens == expected


def test_get_clitic():
    assert azm.scanner.get_clitic('licetque') == ('licet', 'que')
    assert azm.scanner.get_clitic('Estne') == ('Est', 'ne')
    assert azm.scanner.get_clitic('querela') == ('querela', None)