Skip to content

Commit 603853d

Browse files
committed
proxy: tree-sitter to collect metadata
In cases when the underlying ocl lib does not provide metadata, use tree sitter to parse the kernel source and extract the metadata from that. Uses code generated from: https://github.com/lefp/tree-sitter-opencl.
1 parent dc44a03 commit 603853d

14 files changed

+102265
-4
lines changed

CMakeLists.txt

+5-1
Original file line numberDiff line numberDiff line change
@@ -1767,7 +1767,11 @@ if(ENABLE_PROXY_DEVICE)
17671767
set(RENAME_POCL OFF )
17681768
endif()
17691769

1770-
1770+
find_library(TSLIB tree-sitter)
1771+
set(HAVE_TREE_SITTER TSLIB)
1772+
if(TSLIB)
1773+
message("-- FOUND tree sitter library")
1774+
endif()
17711775

17721776
set(BUILD_PROXY 1)
17731777

config.h.in.cmake

+2
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@
112112

113113
#cmakedefine HAVE_OPENCV
114114

115+
#cmakedefine HAVE_TREE_SITTER 1
116+
115117
#cmakedefine HAVE_OCL_ICD
116118
#cmakedefine HAVE_OCL_ICD_30_COMPATIBLE
117119

lib/CL/devices/CMakeLists.txt

+10
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,12 @@ if(ENABLE_TCE)
125125
endif()
126126

127127
if(ENABLE_PROXY_DEVICE)
128+
if(HAVE_TREE_SITTER)
129+
add_subdirectory("tree_sitter")
130+
list(APPEND POCL_DEVICES_LINK_LIST tree-sitter)
131+
set(POCL_DEVICES_OBJS "${POCL_DEVICES_OBJS}"
132+
"$<TARGET_OBJECTS:pocl_tree_sitter_utils>")
133+
endif ()
128134
add_subdirectory("proxy")
129135
set(POCL_DEVICES_OBJS "${POCL_DEVICES_OBJS}"
130136
"$<TARGET_OBJECTS:pocl-devices-proxy>")
@@ -226,6 +232,10 @@ if(HAVE_OPENCV)
226232
target_link_libraries("pocl-devices" PRIVATE opencv_dnn)
227233
endif ()
228234

235+
if(HAVE_TREE_SITTER)
236+
target_link_libraries("pocl-devices" PRIVATE tree-sitter)
237+
endif()
238+
229239
set(POCL_DEVICES_OBJS "${POCL_DEVICES_OBJS}"
230240
"$<TARGET_OBJECTS:pocl-devices>")
231241
harden("pocl-devices")

lib/CL/devices/proxy/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@
2525

2626
add_pocl_device_library("pocl-devices-proxy" pocl_proxy.hpp pocl_proxy.cpp)
2727

28+
if(HAVE_TREE_SITTER)
29+
target_link_libraries("pocl-devices-proxy" PRIVATE pocl_tree_sitter_utils)
30+
endif()
31+
2832
if(PROXY_USE_LIBOPENCL_STUB)
2933

3034
set(STUB_SOURCES libopencl_stub/openclstub.c libopencl_stub/openclstub.h libopencl_stub/rename_stub.h)

lib/CL/devices/proxy/pocl_proxy.cpp

+141-3
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@
5858
#error This driver cannot be built when pocl is to be linked against ICD
5959
#endif
6060

61+
#ifdef HAVE_TREE_SITTER
62+
#include "bindings/c/tree-sitter-opencl.h"
63+
#include "pocl_tree_sitter_utils.h"
64+
#endif
65+
6166
/*****************************************************************************/
6267

6368
typedef struct proxy_platform_data_s
@@ -983,6 +988,120 @@ static int get_kernel_metadata(pocl_kernel_metadata_t *meta,
983988
return CL_SUCCESS;
984989
}
985990

991+
#ifdef HAVE_TREE_SITTER
992+
/**
993+
* Use tree sitter to populate metadata from the kernel source string.
994+
*
995+
* @param source [in]: String of kernel source.
996+
* @param meta [out]: Populated with metadata of kernel.
997+
* @param num_devices [in]: device count.
998+
* @param device [in]: used to populate metadata.
999+
* @return CL_SUCCESS or a CL error.
1000+
*/
1001+
static int proxy_map_source_to_metadata(const char *source,
1002+
pocl_kernel_metadata_t *meta,
1003+
cl_uint num_devices,
1004+
cl_device_id device, cl_kernel kernel) {
1005+
1006+
char string_value[POCL_MAX_PATHNAME_LENGTH];
1007+
int err;
1008+
size_t size;
1009+
1010+
assert(meta->data == nullptr);
1011+
meta->data = (void **)calloc(num_devices, sizeof(void *));
1012+
meta->has_arg_metadata = (-1);
1013+
1014+
err = clGetKernelInfo(kernel, CL_KERNEL_FUNCTION_NAME, 0, nullptr, &size);
1015+
if (err != CL_SUCCESS || size == 0) {
1016+
POCL_MSG_PRINT_PROXY("Could not get kernel name.\n");
1017+
return err;
1018+
}
1019+
assert(size < POCL_MAX_PATHNAME_LENGTH);
1020+
1021+
err = clGetKernelInfo(kernel, CL_KERNEL_FUNCTION_NAME, size, string_value,
1022+
nullptr);
1023+
if (err != CL_SUCCESS) {
1024+
POCL_MSG_PRINT_PROXY("Could not copy kernel name.\n");
1025+
return err;
1026+
}
1027+
meta->name = (char *)malloc(size);
1028+
memcpy(meta->name, string_value, size);
1029+
1030+
err = get_kernel_info(meta, device, kernel);
1031+
if (err != CL_SUCCESS) {
1032+
POCL_MSG_PRINT_PROXY("Could not get kernel info.\n");
1033+
return err;
1034+
}
1035+
1036+
cl_uint num_args;
1037+
1038+
err = clGetKernelInfo(kernel, CL_KERNEL_NUM_ARGS, sizeof(num_args), &num_args,
1039+
nullptr);
1040+
if (err != CL_SUCCESS) {
1041+
POCL_MSG_PRINT_PROXY("Could not get number of kernel args.\n");
1042+
return err;
1043+
}
1044+
1045+
if (num_args == 0) {
1046+
meta->arg_info = nullptr;
1047+
meta->num_args = 0;
1048+
return CL_SUCCESS;
1049+
}
1050+
1051+
assert(num_args < 10000);
1052+
1053+
meta->num_args = num_args;
1054+
meta->arg_info =
1055+
(pocl_argument_info *)calloc(num_args, sizeof(pocl_argument_info));
1056+
1057+
char empty_buffer[MAX_TESTED_ARG_SIZE];
1058+
1059+
TSParser *parser = ts_parser_new();
1060+
ts_parser_set_language(parser, tree_sitter_opencl());
1061+
TSTree *tree =
1062+
ts_parser_parse_string(parser, nullptr, source, strlen(source));
1063+
TSNode root_node = ts_tree_root_node(tree);
1064+
1065+
int32_t status = 0;
1066+
TSNode kernel_node =
1067+
pocl_ts_find_kernel_params(source, root_node, meta->name, &status);
1068+
uint32_t child_count;
1069+
if (status != CL_SUCCESS) {
1070+
err = CL_KERNEL_ARG_INFO_NOT_AVAILABLE;
1071+
POCL_MSG_WARN("Could not find kernel name in source.\n");
1072+
goto TS_ERROR;
1073+
}
1074+
POCL_PRINT_TS_NODE(source, kernel_node);
1075+
child_count = ts_node_named_child_count(kernel_node);
1076+
assert(num_args == child_count);
1077+
1078+
TSNode child;
1079+
for (cl_uint i = 0; i < num_args; ++i) {
1080+
pocl_argument_info *pi = &meta->arg_info[i];
1081+
child = ts_node_named_child(kernel_node, i);
1082+
err = pocl_ts_map_node_to_arg_info(source, child, pi);
1083+
if (err != CL_SUCCESS) {
1084+
POCL_MSG_WARN("Failed to parse %s arg %d.\n", meta->name, i);
1085+
goto TS_ERROR;
1086+
}
1087+
1088+
POCL_MSG_PRINT_PROXY("KERNEL %s ARGUMENT %u NAME %s "
1089+
"TYPENAME %s TYPE %u SIZE %u\n",
1090+
meta->name, i, pi->name, pi->type_name, pi->type,
1091+
pi->type_size);
1092+
}
1093+
1094+
ts_tree_delete(tree);
1095+
ts_parser_delete(parser);
1096+
return CL_SUCCESS;
1097+
1098+
TS_ERROR:
1099+
ts_tree_delete(tree);
1100+
ts_parser_delete(parser);
1101+
return err;
1102+
}
1103+
#endif
1104+
9861105
static void
9871106
set_build_log (cl_device_id proxy_dev, cl_program program,
9881107
cl_program proxy_prog, unsigned device_i)
@@ -1401,8 +1520,8 @@ pocl_proxy_setup_metadata (cl_device_id device, cl_program program,
14011520
cl_program proxy_prog = (cl_program)program->data[program_device_i];
14021521

14031522
// Return if there is no metadata and we are not building using a binary.
1404-
if (!(d->backend->provides_metadata ||
1405-
(d->backend->supports_il && program->program_il_size > 0)))
1523+
if (!(d->backend->provides_metadata || HAVE_TREE_SITTER ||
1524+
(d->backend->supports_il && program->program_il_size > 0)))
14061525
return 0;
14071526

14081527
assert(program->kernel_meta == NULL);
@@ -1471,8 +1590,9 @@ pocl_proxy_setup_metadata (cl_device_id device, cl_program program,
14711590
index++;
14721591
}
14731592

1474-
} else {
1593+
} else if (d->backend->provides_metadata) {
14751594
for (cl_uint i = 0; i < num_kernels; ++i) {
1595+
14761596
err = get_kernel_metadata(p + i, program->num_devices, proxy_prog,
14771597
d->device_id, kernels[i]);
14781598
if (err != CL_SUCCESS) {
@@ -1484,6 +1604,24 @@ pocl_proxy_setup_metadata (cl_device_id device, cl_program program,
14841604
assert(err == CL_SUCCESS);
14851605
}
14861606
}
1607+
#ifdef HAVE_TREE_SITTER
1608+
else if (HAVE_TREE_SITTER) {
1609+
for (cl_uint i = 0; i < num_kernels; i++) {
1610+
1611+
err = proxy_map_source_to_metadata(program->source, p + i,
1612+
program->num_devices, d->device_id,
1613+
kernels[i]);
1614+
if (err != CL_SUCCESS) {
1615+
POCL_MSG_WARN("Failed to kernel metadata for index %d.\n", i);
1616+
return 0;
1617+
}
1618+
1619+
err = clReleaseKernel(kernels[i]);
1620+
}
1621+
}
1622+
#endif
1623+
else
1624+
return 0;
14871625

14881626
program->kernel_meta = p;
14891627

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#=============================================================================
2+
# CMake build system files
3+
#
4+
# Copyright (c) 2014-2024 pocl developers
5+
#
6+
# Permission is hereby granted, free of charge, to any person obtaining a copy
7+
# of this software and associated documentation files (the "Software"), to deal
8+
# in the Software without restriction, including without limitation the rights
9+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
# copies of the Software, and to permit persons to whom the Software is
11+
# furnished to do so, subject to the following conditions:
12+
#
13+
# The above copyright notice and this permission notice shall be included in
14+
# all copies or substantial portions of the Software.
15+
#
16+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22+
# THE SOFTWARE.
23+
#
24+
#=============================================================================
25+
26+
add_library(pocl_tree_sitter_utils OBJECT pocl_tree_sitter_utils.c
27+
pocl_tree_sitter_utils.h src/parser.c bindings/c/tree-sitter-opencl.h)
28+
29+
target_include_directories(pocl_tree_sitter_utils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
30+
31+
target_link_libraries(pocl_tree_sitter_utils PUBLIC tree-sitter)

lib/CL/devices/tree_sitter/LICENSE

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
The MIT License (MIT)
2+
3+
Copyright (c) 2014 Max Brunsfeld (C version), 2023 Peter Lef (OpenCL C modifications).
4+
5+
Generated from https://github.com/lefp/tree-sitter-opencl commit: 8e1d24a57066
6+
7+
Permission is hereby granted, free of charge, to any person obtaining a copy
8+
of this software and associated documentation files (the "Software"), to deal
9+
in the Software without restriction, including without limitation the rights
10+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
copies of the Software, and to permit persons to whom the Software is
12+
furnished to do so, subject to the following conditions:
13+
14+
The above copyright notice and this permission notice shall be included in all
15+
copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23+
SOFTWARE.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
The MIT License (MIT)
3+
4+
Copyright (c) 2014 Max Brunsfeld (C version), 2023 Peter Lef (OpenCL C
5+
modifications).
6+
7+
Generated from https://github.com/lefp/tree-sitter-opencl commit: 8e1d24a57066
8+
9+
Permission is hereby granted, free of charge, to any person obtaining a copy
10+
of this software and associated documentation files (the "Software"), to deal
11+
in the Software without restriction, including without limitation the rights
12+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
copies of the Software, and to permit persons to whom the Software is
14+
furnished to do so, subject to the following conditions:
15+
16+
The above copyright notice and this permission notice shall be included in all
17+
copies or substantial portions of the Software.
18+
19+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25+
SOFTWARE.
26+
*/
27+
28+
#ifndef TREE_SITTER_OPENCL_H_
29+
#define TREE_SITTER_OPENCL_H_
30+
31+
typedef struct TSLanguage TSLanguage;
32+
33+
#ifdef __cplusplus
34+
extern "C"
35+
{
36+
#endif
37+
38+
const TSLanguage *tree_sitter_opencl (void);
39+
40+
#ifdef __cplusplus
41+
}
42+
#endif
43+
44+
#endif // TREE_SITTER_OPENCL_H_

0 commit comments

Comments
 (0)