Merge pull request pocl#1712 from franz/misc_fixes

Misc fixes

Author: pjaaskel

CMakeLists.txt

@@ -1483,7 +1483,7 @@ set(HOST_DEVICE_CL_VERSION_MINOR 0)
 # Host CPU device: list of extensions that are always enabled, for both OpenCL 1.2 and 3.0
 set(HOST_DEVICE_EXTENSIONS "cl_khr_byte_addressable_store cl_khr_global_int32_base_atomics \
 cl_khr_global_int32_extended_atomics cl_khr_local_int32_base_atomics \
-cl_khr_local_int32_extended_atomics cl_khr_3d_image_writes \
+cl_khr_local_int32_extended_atomics cl_khr_3d_image_writes cl_ext_float_atomics \
 cl_intel_unified_shared_memory cl_ext_buffer_device_address")
 
 # Host CPU device: list of OpenCL 3.0 features that are always enabled
@@ -1519,12 +1519,12 @@ if(NOT ENABLE_CONFORMANCE)
     set(HOST_DEVICE_EXTENSIONS "${HOST_DEVICE_EXTENSIONS} \
       cl_exp_tensor cl_exp_defined_builtin_kernels")
   endif()
-  set(HOST_DEVICE_EXTENSIONS "${HOST_DEVICE_EXTENSIONS} cl_khr_subgroups cl_khr_subgroup_ballot \
-cl_khr_subgroup_shuffle cl_intel_subgroups cl_intel_subgroups_short cl_intel_subgroups_char \
-cl_ext_float_atomics cl_intel_required_subgroup_size")
+  set(HOST_DEVICE_EXTENSIONS "${HOST_DEVICE_EXTENSIONS} \
+cl_khr_subgroups cl_khr_subgroup_ballot cl_khr_subgroup_shuffle \
+cl_intel_subgroups cl_intel_subgroups_short cl_intel_subgroups_char cl_intel_required_subgroup_size")
   # read-write images are still partially broken
   # program-scope variables work on OpenCL C input, but breaks with SPIR-V input (llvm-spirv bugs)
-  set(HOST_DEVICE_FEATURES_30  "${HOST_DEVICE_FEATURES_30} __opencl_c_read_write_images __opencl_c_program_scope_global_variables")
+  set(HOST_DEVICE_FEATURES_30  "${HOST_DEVICE_FEATURES_30} __opencl_c_subgroups __opencl_c_read_write_images __opencl_c_program_scope_global_variables")
 endif()
 
 # Extensions that are considered feature-complete (preferably CTS-tested).

examples/CMakeLists.txt

@@ -68,7 +68,7 @@ endif()
 
 # TODO:   opencl-book-samples
 set(ALL_TESTSUITES
-    AMD AMDSDK2.9 AMDSDK3.0
+    AMDSDK3.0
     ASL arrayfire clBLAS CLBlast clFFT chipStar
     conformance CloverLeaf Glow Halide IntelSVM
     intel-compute-samples oneapi-samples

include/_builtin_renames.h

@@ -1076,4 +1076,67 @@
 #define write_imagei _cl_write_imagei
 #define write_imagef _cl_write_imagef
 
+#define get_sub_group_size _cl_get_sub_group_size
+#define get_max_sub_group_size _cl_get_max_sub_group_size
+#define get_num_sub_groups _cl_get_num_sub_groups
+#define get_enqueued_num_sub_groups _cl_get_enqueued_num_sub_groups
+#define get_sub_group_id _cl_get_sub_group_id
+#define get_sub_group_local_id _cl_get_sub_group_local_id
+#define sub_group_ballot _cl_sub_group_ballot
+#define sub_group_broadcast _cl_sub_group_broadcast
+#define sub_group_barrier _cl_sub_group_barrier
+#define sub_group_any _cl_sub_group_any
+#define sub_group_all _cl_sub_group_all
+#define sub_group_reduce_add _cl_sub_group_reduce_add
+#define sub_group_reduce_min _cl_sub_group_reduce_min
+#define sub_group_reduce_max _cl_sub_group_reduce_max
+#define sub_group_shuffle _cl_sub_group_shuffle
+#define sub_group_shuffle_xor _cl_sub_group_shuffle_xor
+#define sub_group_shuffle_up _cl_sub_group_shuffle_up
+#define sub_group_shuffle_down _cl_sub_group_shuffle_down
+#define intel_sub_group_shuffle _cl_intel_sub_group_shuffle
+#define intel_sub_group_shuffle_xor _cl_intel_sub_group_shuffle_xor
+#define intel_sub_group_shuffle_up _cl_intel_sub_group_shuffle_up
+#define intel_sub_group_shuffle_down _cl_intel_sub_group_shuffle_down
+#define sub_group_scan_inclusive_add _cl_sub_group_scan_inclusive_add
+#define sub_group_scan_inclusive_min _cl_sub_group_scan_inclusive_min
+#define sub_group_scan_inclusive_max _cl_sub_group_scan_inclusive_max
+#define sub_group_scan_exclusive_add _cl_sub_group_scan_exclusive_add
+#define sub_group_scan_exclusive_min _cl_sub_group_scan_exclusive_min
+#define sub_group_scan_exclusive_max _cl_sub_group_scan_exclusive_max
+
+#define intel_sub_group_block_read _cl_intel_sub_group_block_read
+#define intel_sub_group_block_read2 _cl_intel_sub_group_block_read2
+#define intel_sub_group_block_read4 _cl_intel_sub_group_block_read4
+#define intel_sub_group_block_read8 _cl_intel_sub_group_block_read8
+#define intel_sub_group_block_read_uc _cl_intel_sub_group_block_read_uc
+#define intel_sub_group_block_read_uc2 _cl_intel_sub_group_block_read_uc2
+#define intel_sub_group_block_read_uc4 _cl_intel_sub_group_block_read_uc4
+#define intel_sub_group_block_read_uc8 _cl_intel_sub_group_block_read_uc8
+#define intel_sub_group_block_read_us _cl_intel_sub_group_block_read_us
+#define intel_sub_group_block_read_us2 _cl_intel_sub_group_block_read_us2
+#define intel_sub_group_block_read_us4 _cl_intel_sub_group_block_read_us4
+#define intel_sub_group_block_read_us8 _cl_intel_sub_group_block_read_us8
+#define intel_sub_group_block_read_ui _cl_intel_sub_group_block_read_ui
+#define intel_sub_group_block_read_ui2 _cl_intel_sub_group_block_read_ui2
+#define intel_sub_group_block_read_ui4 _cl_intel_sub_group_block_read_ui4
+#define intel_sub_group_block_read_ui8 _cl_intel_sub_group_block_read_ui8
+
+#define intel_sub_group_block_write _cl_intel_sub_group_block_write
+#define intel_sub_group_block_write2 _cl_intel_sub_group_block_write2
+#define intel_sub_group_block_write4 _cl_intel_sub_group_block_write4
+#define intel_sub_group_block_write8 _cl_intel_sub_group_block_write8
+#define intel_sub_group_block_write_uc _cl_intel_sub_group_block_write_uc
+#define intel_sub_group_block_write_uc2 _cl_intel_sub_group_block_write_uc2
+#define intel_sub_group_block_write_uc4 _cl_intel_sub_group_block_write_uc4
+#define intel_sub_group_block_write_uc8 _cl_intel_sub_group_block_write_uc8
+#define intel_sub_group_block_write_us _cl_intel_sub_group_block_write_us
+#define intel_sub_group_block_write_us2 _cl_intel_sub_group_block_write_us2
+#define intel_sub_group_block_write_us4 _cl_intel_sub_group_block_write_us4
+#define intel_sub_group_block_write_us8 _cl_intel_sub_group_block_write_us8
+#define intel_sub_group_block_write_ui _cl_intel_sub_group_block_write_ui
+#define intel_sub_group_block_write_ui2 _cl_intel_sub_group_block_write_ui2
+#define intel_sub_group_block_write_ui4 _cl_intel_sub_group_block_write_ui4
+#define intel_sub_group_block_write_ui8 _cl_intel_sub_group_block_write_ui8
+
 #endif

lib/CL/clCreateCommandBufferKHR.c

@@ -64,7 +64,7 @@ POname (clCreateCommandBufferKHR) (
   cl_uint num_properties = 0;
   cl_command_buffer_properties_khr *seen_keys = NULL;
   cl_bool property_mutable = 0;
-  cl_bool property_asserts = 0;
+  cl_bool property_assert_no_more_wgs = 0;
   if (properties != NULL)
     {
       const cl_command_buffer_properties_khr *key = 0;
@@ -116,7 +116,10 @@ POname (clCreateCommandBufferKHR) (
               break;
 
             case CL_COMMAND_BUFFER_MUTABLE_DISPATCH_ASSERTS_KHR:
-              property_asserts = (val & CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR) > 0;
+              property_assert_no_more_wgs
+                = (val
+                   & CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR)
+                  > 0;
               break;
 
             default:
@@ -154,6 +157,7 @@ POname (clCreateCommandBufferKHR) (
   cmdbuf->state = CL_COMMAND_BUFFER_STATE_RECORDING_KHR;
   cmdbuf->num_queues = num_queues;
   cmdbuf->is_mutable = property_mutable;
+  cmdbuf->assert_no_more_wgs = property_assert_no_more_wgs;
   cmdbuf->queues
       = (cl_command_queue *)calloc (num_queues, sizeof (cl_command_queue));
   memcpy (cmdbuf->queues, queues, num_queues * sizeof (cl_command_queue));

lib/CL/devices/common_utils.c

@@ -349,6 +349,24 @@ pocl_cpu_init_common (cl_device_id device)
   device->wg_collective_func_support = CL_TRUE;
   device->device_side_printf = CL_TRUE;
 
+  if (strstr (HOST_DEVICE_EXTENSIONS, "cl_ext_float_atomics") != NULL)
+    {
+      device->single_fp_atomic_caps = device->double_fp_atomic_caps
+        = CL_DEVICE_GLOBAL_FP_ATOMIC_ADD_EXT
+          | CL_DEVICE_GLOBAL_FP_ATOMIC_MIN_MAX_EXT
+          | CL_DEVICE_LOCAL_FP_ATOMIC_ADD_EXT
+          | CL_DEVICE_LOCAL_FP_ATOMIC_MIN_MAX_EXT;
+      device->features
+        = HOST_DEVICE_FEATURES_30 " __opencl_c_ext_fp32_global_atomic_add "
+                                  "__opencl_c_ext_fp64_global_atomic_add"
+                                  " __opencl_c_ext_fp32_local_atomic_add "
+                                  "__opencl_c_ext_fp64_local_atomic_add"
+                                  " __opencl_c_ext_fp32_global_atomic_min_max "
+                                  "__opencl_c_ext_fp64_global_atomic_min_max"
+                                  " __opencl_c_ext_fp32_local_atomic_min_max "
+                                  "__opencl_c_ext_fp64_local_atomic_min_max";
+    }
+
   pocl_setup_opencl_c_with_version (device, CL_TRUE);
   pocl_setup_features_with_version (device);
 
@@ -385,18 +403,6 @@ pocl_cpu_init_common (cl_device_id device)
                      | CL_DEVICE_SVM_FINE_GRAIN_BUFFER
                      | CL_DEVICE_SVM_FINE_GRAIN_SYSTEM
                      | CL_DEVICE_SVM_ATOMICS;
-
-  if (strstr (HOST_DEVICE_EXTENSIONS, "cl_ext_float_atomics") != NULL)
-    {
-      device->single_fp_atomic_caps = device->double_fp_atomic_caps
-          = CL_DEVICE_GLOBAL_FP_ATOMIC_LOAD_STORE_EXT
-            | CL_DEVICE_GLOBAL_FP_ATOMIC_ADD_EXT
-            | CL_DEVICE_GLOBAL_FP_ATOMIC_MIN_MAX_EXT
-            | CL_DEVICE_LOCAL_FP_ATOMIC_LOAD_STORE_EXT
-            | CL_DEVICE_LOCAL_FP_ATOMIC_ADD_EXT
-            | CL_DEVICE_LOCAL_FP_ATOMIC_MIN_MAX_EXT;
-    }
-
 #endif
 
   if (strstr (HOST_DEVICE_EXTENSIONS, "cl_intel_unified_shared_memory")

lib/CL/pocl_cl.h

@@ -1578,6 +1578,7 @@ struct _cl_command_buffer_khr
 
   _cl_command_node *cmds;
   cl_bool is_mutable;
+  cl_bool assert_no_more_wgs;
 };
 
 #define POCL_ON_SUB_MISALIGN(mem, que, operation)                             \

lib/CL/pocl_llvm_utils.cc

@@ -538,7 +538,20 @@ void InitializeLLVM() {
       O = opts["debug"];
       assert(O && "could not find LLVM option 'debug'");
       O->addOccurrence(1, StringRef("debug"), StringRef("true"), false);
+#if 0
+      O = opts["debug-only"];
+      assert(O && "could not find LLVM option 'debug'");
+      O->addOccurrence(1, StringRef("debug-only"), StringRef("inline"), false);
+#endif
     }
+    O = opts["inline-threshold"];
+    assert(O && "inline-threshold not found");
+    O->addOccurrence(1, StringRef("inline-threshold"), StringRef("1200"));
+#if 0
+    O = opts["inline-enable-cost-benefit-analysis"];
+    assert(O && "inline-enable-cost-benefit-analysis not found");
+    O->addOccurrence(1, StringRef("inline-enable-cost-benefit-analysis"), StringRef("true"));
+#endif
   }
 }
 

lib/kernel/SPIR/generate_spir_wrapper.py

@@ -757,6 +757,10 @@ def generate_function(name, ret_type, ret_type_ext, multiAS, *args):
 			llvm_i += 1
 
 		####### process args
+		if not args:
+			spir_mangled_func_suffix.append("v")
+			ocl_mangled_func_suffix.append("v")
+			args = []
 		for cast in args:
 
 			if arg_addr_spaces:
@@ -856,7 +860,6 @@ def generate_function(name, ret_type, ret_type_ext, multiAS, *args):
 				decl_args.append(ocl_arg_type)
 			arg_i += 1
 
-
 		######## generate final mangled function names
 		spir_mangled_func_suffix = "".join(spir_mangled_func_suffix)
 		ocl_mangled_func_suffix = "".join(ocl_mangled_func_suffix)
@@ -873,7 +876,7 @@ def generate_function(name, ret_type, ret_type_ext, multiAS, *args):
 			decl_ret_type = 'void'
 		else:
 			decl_ret_type = coerced_ret_type
-		
+
 		#if GENERIC_AS and GEN_AS_CALLEE_IDENTICAL and (AS == "generic") and ("private" in addr_spaces):
 		declaration = "declare %s %s(%s) local_unnamed_addr #0" % (decl_ret_type, ocl_mangled_name, decl_args)
 		if declaration in ALREADY_DECLARED.keys():
@@ -1300,10 +1303,9 @@ def generate_function(name, ret_type, ret_type_ext, multiAS, *args):
 			generate_function("shuffle2", SIG_TO_LLVM_TYPE_MAP[ret_type], '', False, in_type, in_type, mask_type)
 
 # convert
+CONVERT_TYPES = ['c', 'h', 's', 't', 'i', 'j', 'l', 'm', 'f', 'd']
 if FP16:
-	CONVERT_TYPES = ['c', 'h', 's', 't', 'i', 'j', 'l', 'm', 'f', 'd', 'Dh']
-else:
-	CONVERT_TYPES = ['c', 'h', 's', 't', 'i', 'j', 'l', 'm', 'f', 'd']
+	CONVERT_TYPES.append('Dh')
 
 for dst_type in CONVERT_TYPES:
 	for src_type in CONVERT_TYPES:
@@ -1494,6 +1496,77 @@ def gen_three_variants(f, ret_type, ret_ext, AS, args, orders):
 generate_function("work_group_barrier", SIG_TO_LLVM_TYPE_MAP['v'], '', None, 'j', '12memory_scope')
 generate_function("work_group_barrier", SIG_TO_LLVM_TYPE_MAP['v'], '', None, 'j')
 
+# generate wrapper function
+#def generate_function(name, ret_type, ret_type_ext, multiAS, *args):
+# subgroups
+generate_function("get_sub_group_size", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("get_max_sub_group_size", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("get_num_sub_groups", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("get_enqueued_num_sub_groups", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("get_sub_group_id", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("get_sub_group_local_id", SIG_TO_LLVM_TYPE_MAP['i'], '', None)
+generate_function("sub_group_ballot", SIG_TO_LLVM_TYPE_MAP['Dv4_i'], '', None, 'i')
+generate_function("sub_group_any", SIG_TO_LLVM_TYPE_MAP['i'], '', None, 'i')
+generate_function("sub_group_all", SIG_TO_LLVM_TYPE_MAP['i'], '', None, 'i')
+generate_function("sub_group_barrier", SIG_TO_LLVM_TYPE_MAP['v'], '', None, 'j')
+generate_function("sub_group_barrier", SIG_TO_LLVM_TYPE_MAP['v'], '', None, 'j', '12memory_scope')
+
+SUBGROUP_TYPES = ['c', 'h', 's', 't', 'i', 'j', 'l', 'm', 'f', 'd']
+if FP16:
+	SUBGROUP_TYPES.append('Dh')
+
+for arg_type in SUBGROUP_TYPES:
+	ret_type = arg_type
+	signext = LLVM_TYPE_EXT_MAP[ret_type]
+	mask_type = 'j'
+	for suffix in ['', '_xor']:
+		generate_function("sub_group_shuffle"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type, mask_type)
+		generate_function("intel_sub_group_shuffle"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type, mask_type)
+	generate_function("sub_group_broadcast", SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type, mask_type)
+	for suffix in ['_add', '_min', '_max']:
+		generate_function("sub_group_reduce"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type)
+		generate_function("sub_group_scan_inclusive"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type)
+		generate_function("sub_group_scan_exclusive"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type)
+	for suffix in ['_up', '_down']:
+		generate_function("intel_sub_group_shuffle"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], signext, False, arg_type, arg_type, mask_type)
+
+# Intel extension adds support for some types which are not supported by the Khronos extension:
+# For the sub_group_shuffle, sub_group_shuffle_down, sub_group_shuffle_up, and sub_group_shuffle_xor functions, gentype is float, float2, float3,
+# float4, float8, float16, int, int2, int3, int4, int8, int16, uint, uint2, uint3, uint4, uint8, uint16, long, or ulong.
+SUBGROUP_VEC_TYPES = ['f','i','j']
+for int_type in SUBGROUP_VEC_TYPES:
+	for vecsize in ['2','3','4','8','16']:
+		ret_type = arg_type = 'Dv'+vecsize+'_'+int_type
+		mask_type = 'j'
+		for suffix in ['', '_xor']:
+			generate_function("intel_sub_group_shuffle"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], '', False, arg_type, mask_type)
+		for suffix in ['_up', '_down']:
+			generate_function("intel_sub_group_shuffle"+suffix, SIG_TO_LLVM_TYPE_MAP[ret_type], '', False, arg_type, arg_type, mask_type)
+
+for vecsize in ['','2','4','8']:
+	# uints
+	if vecsize:
+		arg_type = 'Dv' + vecsize + '_j'
+	else:
+		arg_type = 'j'
+	ret_type = arg_type
+	PConstArg = 'PKj'
+	PArg = 'Pj'
+	generate_function("intel_sub_group_block_read"+vecsize, SIG_TO_LLVM_TYPE_MAP[ret_type], '', ("global"), PConstArg)
+	generate_function("intel_sub_group_block_write"+vecsize, SIG_TO_LLVM_TYPE_MAP['v'], '', ("global", 'none'), PArg, arg_type)
+	generate_function("intel_sub_group_block_read_ui"+vecsize, SIG_TO_LLVM_TYPE_MAP[ret_type], '', ("global"), PConstArg)
+	generate_function("intel_sub_group_block_write_ui"+vecsize, SIG_TO_LLVM_TYPE_MAP['v'], '', ("global", 'none'), PArg, arg_type)
+	# ushorts
+	if vecsize:
+		arg_type = 'Dv' + vecsize + '_t'
+	else:
+		arg_type = 't'
+	ret_type = arg_type
+	PConstArg = 'PKt'
+	PArg = 'Pt'
+	generate_function("intel_sub_group_block_read_us"+vecsize, SIG_TO_LLVM_TYPE_MAP[ret_type], '', ("global"), PConstArg)
+	generate_function("intel_sub_group_block_write_us"+vecsize, SIG_TO_LLVM_TYPE_MAP['v'], '', ("global", 'none'), PArg, arg_type)
+
 print("""
 
 attributes #0 = { alwaysinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }

lib/kernel/host/CMakeLists.txt

@@ -300,7 +300,7 @@ endif()
 
 
 if(HOST_DEVICE_EXTENSIONS MATCHES "cl_khr_subgroup")
-  list(APPEND SOURCES_WITH_SLEEF subgroups.c subgroups.cl)
+  list(APPEND SOURCES_WITH_SLEEF subgroups.cl)
 endif()