New lexers

elixir/filters/__init__.py

@@ -1,23 +1,51 @@
-from typing import List
-
-from .utils import Filter, FilterContext
-from .projects import project_filters, default_filters
-
-# Returns a list of applicable filters for project_name under provided filter context
-def get_filters(ctx: FilterContext, project_name: str) -> List[Filter]:
-    filter_classes = project_filters.get(project_name, default_filters)
-    filters = []
-
-    for filter_cls in filter_classes:
-        if type(filter_cls) == tuple and len(filter_cls) == 2:
-            cls, kwargs = filter_cls
-            filters.append(cls(**kwargs))
-        elif type(filter_cls) == type:
-            filters.append(filter_cls())
-        else:
-            raise ValueError(f"Invalid filter: {filter_cls}, " \
-                    "should be either a two element tuple or a type. " \
-                    "Make sure project_filters in project.py is valid.")
-
-    return [f for f in filters if f.check_if_applies(ctx)]
+from .ident import IdentFilter
+
+from .cppinc import CppIncFilter
+from .cpppathinc import CppPathIncFilter
+
+from .defconfig import DefConfigIdentsFilter
+from .configin import ConfigInFilter
+
+from .kconfig import KconfigFilter
+from .kconfigidents import KconfigIdentsFilter
+
+from .dtsi import DtsiFilter
+from .dtscompdocs import DtsCompDocsFilter
+from .dtscompcode import DtsCompCodeFilter
+from .dtscompdts import DtsCompDtsFilter
+
+from .makefileo import MakefileOFilter
+from .makefiledtb import MakefileDtbFilter
+from .makefiledir import MakefileDirFilter
+from .makefilesubdir import MakefileSubdirFilter
+from .makefilefile import MakefileFileFilter
+from .makefilesrctree import MakefileSrcTreeFilter
+from .makefilesubdir import MakefileSubdirFilter
+
+
+# List of filters applied to all projects
+default_filters = [
+    DtsCompCodeFilter,
+    DtsCompDtsFilter,
+    DtsCompDocsFilter,
+    IdentFilter,
+    CppIncFilter,
+]
+
+# List of filters for Kconfig files
+common_kconfig_filters = [
+    KconfigFilter,
+    KconfigIdentsFilter,
+    DefConfigIdentsFilter,
+]
+
+# List of filters for Makefiles
+common_makefile_filters = [
+    MakefileOFilter,
+    MakefileDtbFilter,
+    MakefileDirFilter,
+    MakefileFileFilter,
+    MakefileSubdirFilter,
+    MakefileSrcTreeFilter,
+]
 

elixir/lexers/__init__.py

@@ -0,0 +1,10 @@
+from .lexers import *
+
+default_lexers = {
+    r'.*\.(c|h|cpp|hpp|c++|cxx|cc)': CLexer,
+    r'makefile\..*':  MakefileLexer,
+    r'.*\.dts(i)?': DTSLexer,
+    r'.*\.s': GasLexer,
+    r'kconfig.*': KconfigLexer, #TODO negative lookahead for .rst
+}
+

elixir/lexers/__main__.py

@@ -0,0 +1,35 @@
+if __name__ == "__main__":
+    import sys
+    from . import lexers
+
+    if not (len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[1] == '-s')):
+        print("usage:", sys.argv[0], "[-s]", "path/to/file")
+        exit(1)
+
+    short = sys.argv[1] == '-s'
+
+    filename = sys.argv[-1]
+
+    with open(filename) as f:
+        if filename.endswith(('.c', '.h', '.cpp', '.hpp')):
+            lexer = lexers.CLexer(f.read())
+        elif filename.endswith(('.dts', '.dtsi')):
+            lexer = lexers.DTSLexer(f.read())
+        elif filename.endswith('Kconfig'):
+            lexer = lexers.KconfigLexer(f.read())
+        elif filename.endswith(('.s', '.S')):
+            lexer = lexers.GasLexer(f.read())
+        elif filename.endswith('Makefile'):
+            lexer = lexers.MakefileLexer(f.read())
+        else:
+            raise Exception("no lexer for filetype")
+
+        for token in lexer.lex():
+            if not short:
+                print(token.line, token.token_type.name, token.span, token.token.encode())
+            else:
+                if token.token_type.name == 'IDENTIFIER' or token.token_type.name == 'STRING':
+                    print(f"|{token.token}|", end='')
+                else:
+                    print(token.token, end='')
+