Skip to content

Commit 980b224

Browse files
fstachurafstachura
committed
lexers: Integrate new lexers with the rest of Elixir
1 parent 068c3dd commit 980b224

File tree

7 files changed

+297
-60
lines changed

7 files changed

+297
-60
lines changed

‎elixir/filters/__init__.py‎

Lines changed: 50 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,51 @@
1-
from typing import List
2-
3-
from .utils import Filter, FilterContext
4-
from .projects import project_filters, default_filters
5-
6-
# Returns a list of applicable filters for project_name under provided filter context
7-
def get_filters(ctx: FilterContext, project_name: str) -> List[Filter]:
8-
filter_classes = project_filters.get(project_name, default_filters)
9-
filters = []
10-
11-
for filter_cls in filter_classes:
12-
if type(filter_cls) == tuple and len(filter_cls) == 2:
13-
cls, kwargs = filter_cls
14-
filters.append(cls(**kwargs))
15-
elif type(filter_cls) == type:
16-
filters.append(filter_cls())
17-
else:
18-
raise ValueError(f"Invalid filter: {filter_cls}, " \
19-
"should be either a two element tuple or a type. " \
20-
"Make sure project_filters in project.py is valid.")
21-
22-
return [f for f in filters if f.check_if_applies(ctx)]
1+
from .ident import IdentFilter
2+
3+
from .cppinc import CppIncFilter
4+
from .cpppathinc import CppPathIncFilter
5+
6+
from .defconfig import DefConfigIdentsFilter
7+
from .configin import ConfigInFilter
8+
9+
from .kconfig import KconfigFilter
10+
from .kconfigidents import KconfigIdentsFilter
11+
12+
from .dtsi import DtsiFilter
13+
from .dtscompdocs import DtsCompDocsFilter
14+
from .dtscompcode import DtsCompCodeFilter
15+
from .dtscompdts import DtsCompDtsFilter
16+
17+
from .makefileo import MakefileOFilter
18+
from .makefiledtb import MakefileDtbFilter
19+
from .makefiledir import MakefileDirFilter
20+
from .makefilesubdir import MakefileSubdirFilter
21+
from .makefilefile import MakefileFileFilter
22+
from .makefilesrctree import MakefileSrcTreeFilter
23+
from .makefilesubdir import MakefileSubdirFilter
24+
25+
26+
# List of filters applied to all projects
27+
default_filters = [
28+
DtsCompCodeFilter,
29+
DtsCompDtsFilter,
30+
DtsCompDocsFilter,
31+
IdentFilter,
32+
CppIncFilter,
33+
]
34+
35+
# List of filters for Kconfig files
36+
common_kconfig_filters = [
37+
KconfigFilter,
38+
KconfigIdentsFilter,
39+
DefConfigIdentsFilter,
40+
]
41+
42+
# List of filters for Makefiles
43+
common_makefile_filters = [
44+
MakefileOFilter,
45+
MakefileDtbFilter,
46+
MakefileDirFilter,
47+
MakefileFileFilter,
48+
MakefileSubdirFilter,
49+
MakefileSrcTreeFilter,
50+
]
2351

‎elixir/lexers/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from .lexers import *
2+
3+
default_lexers = {
4+
r'.*\.(c|h|cpp|hpp|c++|cxx|cc)': CLexer,
5+
r'makefile\..*': MakefileLexer,
6+
r'.*\.dts(i)?': DTSLexer,
7+
r'.*\.s': GasLexer,
8+
r'kconfig.*': KconfigLexer, #TODO negative lookahead for .rst
9+
}
10+

‎elixir/project_utils.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import re
2+
from typing import List
3+
4+
from .filters.utils import Filter, FilterContext
5+
from .filters import default_filters
6+
from .projects import projects
7+
from .lexers import default_lexers
8+
9+
# Returns a list of applicable filters for project_name under provided filter context
10+
def get_filters(ctx: FilterContext, project_name: str) -> List[Filter]:
11+
project_config = projects.get(project_name)
12+
if project_config is None or 'filters' not in project_config:
13+
filter_classes = default_filters
14+
else:
15+
filter_classes = project_config['filters']
16+
17+
filters = []
18+
19+
for filter_cls in filter_classes:
20+
if type(filter_cls) == tuple and len(filter_cls) == 2:
21+
cls, kwargs = filter_cls
22+
filters.append(cls(**kwargs))
23+
elif type(filter_cls) == type:
24+
filters.append(filter_cls())
25+
else:
26+
raise ValueError(f"Invalid filter: {filter_cls}, " \
27+
"should be either a two element tuple or a type. " \
28+
"Make sure project_filters in project.py is valid.")
29+
30+
return [f for f in filters if f.check_if_applies(ctx)]
31+
32+
def get_lexer(path: str, project_name: str):
33+
project_config = projects.get(project_name)
34+
if project_config is None or 'lexers' not in project_config:
35+
lexers = default_lexers
36+
else:
37+
lexers = project_config['lexers']
38+
39+
path = path.lower()
40+
for regex, lexer in lexers.items():
41+
if re.match(regex, path):
42+
if type(lexer) == tuple:
43+
lexer_cls, kwargs = lexer
44+
return lambda code: lexer_cls(code, **kwargs)
45+
else:
46+
return lambda code: lexer(code)
47+

‎elixir/projects.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
from collections import OrderedDict
2+
from .filters import *
3+
from .lexers import *
4+
5+
# Dictionary of custom per-projects settings.
6+
# filters:
7+
# Projects not present in this dictionary only use default_filters.
8+
# Use `*` to unpack filter lists defined above,
9+
# you can pass additional options to filters by putting a Filter
10+
# class and a dictionary with options in a tuple, like this:
11+
# (FilterCls, {"option": True}).
12+
# Check filter files and utils.py for information about available options
13+
projects = {
14+
'amazon-freertos': {
15+
'filters': [
16+
*default_filters,
17+
MakefileSubdirFilter,
18+
],
19+
},
20+
'arm-trusted-firmware': {
21+
'filters': [
22+
*default_filters,
23+
CppPathIncFilter,
24+
],
25+
},
26+
'barebox': {
27+
'filters': [
28+
*default_filters,
29+
DtsiFilter,
30+
*common_kconfig_filters,
31+
CppPathIncFilter,
32+
*common_makefile_filters,
33+
],
34+
},
35+
'coreboot': {
36+
'filters': [
37+
*default_filters,
38+
DtsiFilter,
39+
*common_kconfig_filters,
40+
*common_makefile_filters,
41+
],
42+
},
43+
'linux': {
44+
'filters': [
45+
*default_filters,
46+
DtsiFilter,
47+
*common_kconfig_filters,
48+
*common_makefile_filters,
49+
# include/uapi contains includes to user headers under #ifndef __KERNEL__
50+
# Our solution is to ignore all includes in such paths
51+
(CppPathIncFilter, {"path_exceptions": {'^/include/uapi/.*'}}),
52+
],
53+
'lexers': OrderedDict({
54+
r'.*\.(c|h|cpp|hpp|c++|cxx|cc)': CLexer,
55+
r'makefile\..*': MakefileLexer,
56+
r'.*\.dts(i)?': DTSLexer,
57+
r'kconfig.*': KconfigLexer, #TODO negative lookahead for .rst
58+
59+
r'/arch/alpha/.*\.s': (GasLexer, {"arch": "alpha"}),
60+
r'/arch/arc/.*\.s': (GasLexer, {"arch": "arc"}),
61+
r'/arch/arm/.*\.s': (GasLexer, {"arch": "arm32"}),
62+
r'/arch/csky/.*\.s': (GasLexer, {"arch": "csky"}),
63+
r'/arch/m68k/.*\.s': (GasLexer, {"arch": "m68k"}),
64+
r'/arch/microblaze/.*\.s': (GasLexer, {"arch": "microblaze"}),
65+
r'/arch/mips/.*\.s': (GasLexer, {"arch": "mips"}),
66+
r'/arch/openrisc/.*\.s': (GasLexer, {"arch": "openrisc"}),
67+
r'/arch/parisc/.*\.s': (GasLexer, {"arch": "parisc"}),
68+
r'/arch/s390/.*\.s': (GasLexer, {"arch": "s390"}),
69+
r'/arch/sh/.*\.s': (GasLexer, {"arch": "sh"}),
70+
r'/arch/sparc/.*\.s': (GasLexer, {"arch": "sparc"}),
71+
r'/arch/um/.*\.s': (GasLexer, {"arch": "x86"}),
72+
r'/arch/x86/.*\.s': (GasLexer, {"arch": "x86"}),
73+
r'/arch/xtensa/.*\.s': (GasLexer, {"arch": "xtensa"}),
74+
r'.*\.s': GasLexer,
75+
}),
76+
},
77+
'qemu': {
78+
'filters': [
79+
*default_filters,
80+
*common_kconfig_filters,
81+
],
82+
},
83+
'u-boot': {
84+
'filters': [
85+
*default_filters,
86+
DtsiFilter,
87+
*common_kconfig_filters,
88+
CppPathIncFilter,
89+
*common_makefile_filters,
90+
],
91+
'lexers': OrderedDict({
92+
r'.*\.(c|h|cpp|hpp|c++|cxx|cc)': CLexer,
93+
r'makefile\..*': MakefileLexer,
94+
r'.*\.dts(i)?': DTSLexer,
95+
r'kconfig.*': KconfigLexer, #TODO negative lookahead for .rst
96+
97+
r'/arch/arc/.*\.s': (GasLexer, {"arch": "arc"}),
98+
r'/arch/arm/.*\.s': (GasLexer, {"arch": "arm32"}),
99+
r'/arch/m68k/.*\.s': (GasLexer, {"arch": "m68k"}),
100+
r'/arch/microblaze/.*\.s': (GasLexer, {"arch": "microblaze"}),
101+
r'/arch/mips/.*\.s': (GasLexer, {"arch": "mips"}),
102+
r'/arch/riscv/.*\.s': (GasLexer, {"arch": "riscv"}),
103+
r'/arch/sh/.*\.s': (GasLexer, {"arch": "sh"}),
104+
r'/arch/x86/.*\.s': (GasLexer, {"arch": "x86"}),
105+
r'/arch/sandbox/.*\.s': (GasLexer, {"arch": "x86"}),
106+
r'/arch/xtensa/.*\.s': (GasLexer, {"arch": "xtensa"}),
107+
r'.*\.s': GasLexer,
108+
}),
109+
},
110+
'uclibc-ng': {
111+
'filters': [
112+
*default_filters,
113+
ConfigInFilter,
114+
],
115+
},
116+
'zephyr': {
117+
'filters': [
118+
*default_filters,
119+
DtsiFilter,
120+
*common_kconfig_filters,
121+
CppPathIncFilter,
122+
],
123+
},
124+
}
125+

‎elixir/query.py‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
from .lib import script, scriptLines, decode
2222
from . import lib
2323
from . import data
24-
import os
24+
from .lexers import TokenType
25+
import os, sys
2526
from collections import OrderedDict
2627
from urllib import parse
2728

@@ -172,29 +173,38 @@ def query(self, cmd, *args):
172173

173174
version = args[0]
174175
path = args[1]
176+
lexer = args[2]
175177

176178
filename = os.path.basename(path)
177179
family = lib.getFileFamily(filename)
178180

179-
if family != None:
181+
if family is not None and lexer is not None:
180182
buffer = BytesIO()
181-
tokens = self.scriptLines('tokenize-file', version, path, family)
182-
even = True
183+
code = self.get_file_raw(version, path)
183184

184185
prefix = b''
185186
if family == 'K':
186187
prefix = b'CONFIG_'
187188

188-
for tok in tokens:
189-
even = not even
190-
tok2 = prefix + tok
191-
if (even and self.db.defs.exists(tok2) and
192-
(lib.compatibleFamily(self.db.defs.get(tok2).get_families(), family) or
193-
lib.compatibleMacro(self.db.defs.get(tok2).get_macros(), family))):
194-
tok = b'\033[31m' + tok2 + b'\033[0m'
195-
else:
196-
tok = lib.unescape(tok)
197-
buffer.write(tok)
189+
for token_type, token, _, line in lexer(code).lex():
190+
token = token.encode()
191+
192+
if token_type == TokenType.ERROR:
193+
print("error token: ", token, token_type, filename, line, file=sys.stderr)
194+
elif token_type == TokenType.IDENTIFIER:
195+
token_with_prefix = prefix + token
196+
token_in_db = self.db.defs.exists(token_with_prefix)
197+
if token_in_db:
198+
compatible = \
199+
lib.compatibleFamily(self.db.defs.get(token_with_prefix).get_families(), family) or \
200+
lib.compatibleMacro(self.db.defs.get(token_with_prefix).get_macros(), family)
201+
202+
if compatible:
203+
buffer.write(b'\033[31m' + token_with_prefix + b'\033[0m')
204+
continue
205+
206+
buffer.write(token)
207+
198208
return decode(buffer.getvalue())
199209
else:
200210
return decode(self.script('get-file', version, path))

‎elixir/web.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333

3434
from .lib import validFamily
3535
from .query import Query, SymbolInstance
36-
from .filters import get_filters
36+
from .project_utils import get_filters, get_lexer
3737
from .filters.utils import FilterContext
3838
from .autocomplete import AutocompleteResource
3939
from .api import ApiIdentGetterResource
@@ -485,7 +485,8 @@ def format_code(filename, code):
485485
# version: requested version of the project
486486
# path: path to the file in the repository
487487
def generate_source(q, project, version, path):
488-
code = q.query('file', version, path)
488+
lexer = get_lexer(path, project)
489+
code = q.query('file', version, path, lexer)
489490

490491
_, fname = os.path.split(path)
491492
_, extension = os.path.splitext(fname)

0 commit comments

Comments
 (0)