Newer
Older
# -*- coding: utf-8 -*-
import allzweckmesser as azm
def test_tokenize():
expected = ['clamavit', 'moriens', 'lingua', ':', "'", 'Corinna', ',',
'vale', '!', "'"]
tokens = azm.scanner.tokenize("clamavit moriens lingua: 'Corinna, vale!'")
plain_tokens = [t.text for t in tokens]
assert plain_tokens == expected
def test_get_clitic():
assert azm.scanner.get_clitic('licetque') == ('licet', 'que')
assert azm.scanner.get_clitic('Estne') == ('Est', 'ne')
assert azm.scanner.get_clitic('querela') == ('querela', None)