scanner.py

import re
from typeDef import TypeDefinition


def parse(typeDef, src_code, filter_=None):
    """
    return a list of Token object, generated by the input src_code
    if want to filter out any type of token, use filter.
    eg:
        parse(typedef, src, ['block_comment', 'line_comment'])
    """
    assert isinstance(typeDef, TypeDefinition)
    result = []
    token_regex = typeDef.getRE()
    for mo in re.finditer(token_regex, src_code):
        if filter_ is None or mo.lastgroup not in filter_:
            id_ = typeDef.getID(mo.lastgroup)
            result.append((str(mo[0]), id_))
    result.append(("$", "$"))
    return result


if __name__ == "__main__":
    typedef = TypeDefinition.load("simpleJava/typedef")
    with open("simpleJava/test.sjava", "r") as f:
        src_code = f.read()
    print(parse(typedef, src_code, ['space']))