Newer
Older
# -*- coding: utf-8 -*-
import allzweckmesser as azm
def test_tokenize():
expected = ['clamavit', 'moriens', 'lingua', ':', "'", 'Corinna', ',',
'vale', '!', "'"]
tokens = azm.scanner.tokenize("clamavit moriens lingua: 'Corinna, vale!'")
plain_tokens = [t.text for t in tokens]
assert plain_tokens == expected