test_scanner.py

# -*- coding: utf-8 -*-

import allzweckmesser as azm

def test_tokenize():
    expected = ['clamavit', 'moriens', 'lingua', ':', "'", 'Corinna', ',',
                'vale', '!', "'"]
    tokens = azm.scanner.tokenize("clamavit moriens lingua: 'Corinna, vale!'")
    plain_tokens = [t.text for t in tokens]
    assert plain_tokens == expected