diff --git a/libcudacxx/include/cuda/__mdspan/host_device_accessor.h b/libcudacxx/include/cuda/__mdspan/host_device_accessor.h index 6afac79d74a..d40ef1f09f9 100644 --- a/libcudacxx/include/cuda/__mdspan/host_device_accessor.h +++ b/libcudacxx/include/cuda/__mdspan/host_device_accessor.h @@ -23,10 +23,11 @@ #include #include +#include #include +#include #include #include -#include #include #include #include @@ -35,8 +36,6 @@ #include #include #include -#include -#include #include @@ -105,21 +104,13 @@ class __host_accessor : public _Accessor noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0)); #if !_CCCL_COMPILER(NVRTC) - [[nodiscard]] _CCCL_HOST_API static constexpr bool - __is_host_accessible_pointer([[maybe_unused]] __data_handle_type __p) noexcept + [[nodiscard]] + _CCCL_HOST_API static constexpr bool __is_host_accessible_pointer([[maybe_unused]] __data_handle_type __p) noexcept { # if _CCCL_HAS_CTK() if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>) { - _CCCL_IF_NOT_CONSTEVAL_DEFAULT - { - auto __p1 = ::cuda::std::to_address(__p); - ::CUmemorytype __type{}; - const auto __status = - ::cuda::__driver::__pointerGetAttributeNoThrow<::CU_POINTER_ATTRIBUTE_MEMORY_TYPE>(__type, __p1); - return (__status != ::cudaSuccess) || __type == ::CU_MEMORYTYPE_HOST; - } - return true; + return ::cuda::is_host_accessible(::cuda::std::to_address(__p)); } else # endif // _CCCL_HAS_CTK() @@ -202,25 +193,27 @@ class __host_accessor : public _Accessor : _Accessor{__acc} {} - _CCCL_API constexpr reference access(data_handle_type __p, size_t __i) const noexcept(__is_access_noexcept) + _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const + noexcept(__is_access_noexcept) { - NV_IF_ELSE_TARGET( - NV_IS_DEVICE, - (_CCCL_VERIFY(false, "cuda::__host_accessor cannot be used in DEVICE code");), - (_CCCL_ASSERT(__is_host_accessible_pointer(__p), "cuda::__host_accessor data handle is not a HOST pointer");)) + NV_IF_TARGET(NV_IS_DEVICE, (_CCCL_VERIFY(false, "cuda::__host_accessor cannot be used in DEVICE code");)) return _Accessor::access(__p, __i); } - [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, size_t __i) const + [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const noexcept(__is_offset_noexcept) { return _Accessor::offset(__p, __i); } [[nodiscard]] _CCCL_API constexpr bool - __detectably_invalid([[maybe_unused]] data_handle_type __p, size_t) const noexcept + __detectably_invalid([[maybe_unused]] data_handle_type __p, ::cuda::std::size_t) const noexcept { - NV_IF_ELSE_TARGET(NV_IS_HOST, (return __is_host_accessible_pointer(__p);), (return false;)) + _CCCL_IF_NOT_CONSTEVAL_DEFAULT + { + NV_IF_ELSE_TARGET(NV_IS_HOST, (return __is_host_accessible_pointer(__p);), (return false;)) + } + return true; } }; @@ -242,20 +235,17 @@ class __device_accessor : public _Accessor static constexpr bool __is_offset_noexcept = noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0)); - [[nodiscard]] _CCCL_API static constexpr bool + [[nodiscard]] _CCCL_API static bool __is_device_accessible_pointer_from_host([[maybe_unused]] __data_handle_type __p) noexcept { -#if _CCCL_HAS_CTK() +#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC) if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>) { - auto __p1 = ::cuda::std::to_address(__p); - ::CUmemorytype __type{}; - const auto __status = - ::cuda::__driver::__pointerGetAttributeNoThrow<::CU_POINTER_ATTRIBUTE_MEMORY_TYPE>(__type, __p1); - return (__status != ::cudaSuccess) || __type == ::CU_MEMORYTYPE_DEVICE; + static const auto __dev_id = static_cast(::cuda::__driver::__ctxGetDevice()); + return ::cuda::is_device_accessible(::cuda::std::to_address(__p), ::cuda::device_ref{__dev_id}); } else -#endif // _CCCL_HAS_CTK() +#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC) { return true; // cannot be verified } @@ -263,8 +253,8 @@ class __device_accessor : public _Accessor #if _CCCL_DEVICE_COMPILATION() - [[nodiscard]] _CCCL_HIDE_FROM_ABI _CCCL_DEVICE static constexpr bool - __is_device_accessible_pointer_from_device(__data_handle_type __p) noexcept + [[nodiscard]] + _CCCL_DEVICE_API static constexpr bool __is_device_accessible_pointer_from_device(__data_handle_type __p) noexcept { return ::cuda::device::is_address_from(__p, ::cuda::device::address_space::global) || ::cuda::device::is_address_from(__p, ::cuda::device::address_space::shared) @@ -276,12 +266,6 @@ class __device_accessor : public _Accessor #endif // _CCCL_DEVICE_COMPILATION() - _CCCL_API static constexpr void __check_device_pointer([[maybe_unused]] __data_handle_type __p) noexcept - { - NV_IF_TARGET(NV_IS_HOST, - (_CCCL_ASSERT(__is_device_accessible_pointer_from_host(__p), "The pointer is not device accessible");)) - } - public: using offset_policy = __device_accessor; using data_handle_type = __data_handle_type; @@ -355,24 +339,34 @@ class __device_accessor : public _Accessor : _Accessor{__acc} {} - _CCCL_API constexpr reference access(data_handle_type __p, size_t __i) const noexcept(__is_access_noexcept) + _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const + noexcept(__is_access_noexcept) { - NV_IF_ELSE_TARGET( - NV_IS_DEVICE, - (_CCCL_ASSERT(__is_device_accessible_pointer_from_device(__p), "The pointer is not device accessible");), - (_CCCL_VERIFY(false, "cuda::device_accessor cannot be used in HOST code");)) + _CCCL_IF_NOT_CONSTEVAL_DEFAULT + { + NV_IF_ELSE_TARGET( + NV_IS_DEVICE, + (_CCCL_ASSERT(__is_device_accessible_pointer_from_device(__p), "The pointer is not device accessible");), + (_CCCL_VERIFY(false, "cuda::device_accessor cannot be used in HOST code");)) + } return _Accessor::access(__p, __i); } - [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, size_t __i) const + [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const noexcept(__is_offset_noexcept) { return _Accessor::offset(__p, __i); } - [[nodiscard]] _CCCL_API constexpr bool __detectably_invalid(data_handle_type __p, size_t) const noexcept + [[nodiscard]] _CCCL_API constexpr bool __detectably_invalid(data_handle_type __p, ::cuda::std::size_t) const noexcept { - NV_IF_ELSE_TARGET(NV_IS_HOST, (return __is_device_accessible_pointer_from_host(__p);), (return false;)) + _CCCL_IF_NOT_CONSTEVAL_DEFAULT + { + NV_IF_ELSE_TARGET(NV_IS_HOST, + (return __is_device_accessible_pointer_from_host(__p);), + (return __is_device_accessible_pointer_from_device(__p);)) + } + return true; } }; @@ -396,27 +390,18 @@ class __managed_accessor : public _Accessor [[nodiscard]] _CCCL_API static constexpr bool __is_managed_pointer([[maybe_unused]] __data_handle_type __p) noexcept { -#if _CCCL_HAS_CTK() +#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC) if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>) { - const auto __p1 = ::cuda::std::to_address(__p); - bool __is_managed{}; - const auto __status = - ::cuda::__driver::__pointerGetAttributeNoThrow<::CU_POINTER_ATTRIBUTE_IS_MANAGED>(__is_managed, __p1); - return (__status != ::cudaSuccess) || __is_managed; + return ::cuda::is_managed(::cuda::std::to_address(__p)); } else -#endif // _CCCL_HAS_CTK() +#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC) { return true; // cannot be verified } } - _CCCL_API static constexpr void __check_managed_pointer([[maybe_unused]] __data_handle_type __p) noexcept - { - _CCCL_ASSERT(__is_managed_pointer(__p), "cuda::__managed_accessor data handle is not a MANAGED pointer"); - } - public: using offset_policy = __managed_accessor; using data_handle_type = __data_handle_type; @@ -477,22 +462,25 @@ class __managed_accessor : public _Accessor : _Accessor{::cuda::std::move(__acc)} {} - _CCCL_API constexpr reference access(data_handle_type __p, size_t __i) const noexcept(__is_access_noexcept) + _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const + noexcept(__is_access_noexcept) { - NV_IF_TARGET(NV_IS_HOST, (__check_managed_pointer(__p);)) return _Accessor::access(__p, __i); } - [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, size_t __i) const + [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const noexcept(__is_offset_noexcept) { return _Accessor::offset(__p, __i); } [[nodiscard]] _CCCL_API constexpr bool - __detectably_invalid([[maybe_unused]] data_handle_type __p, size_t) const noexcept + __detectably_invalid([[maybe_unused]] data_handle_type __p, ::cuda::std::size_t) const noexcept { - NV_IF_ELSE_TARGET(NV_IS_HOST, (return __is_managed_pointer(__p);), (return false;)) + _CCCL_IF_NOT_CONSTEVAL_DEFAULT + { + NV_IF_ELSE_TARGET(NV_IS_HOST, (return __is_managed_pointer(__p);), (return true;)) + } } }; diff --git a/libcudacxx/include/cuda/std/__mdspan/concepts.h b/libcudacxx/include/cuda/std/__mdspan/concepts.h index 8b29e3a90bb..02fca20311f 100644 --- a/libcudacxx/include/cuda/std/__mdspan/concepts.h +++ b/libcudacxx/include/cuda/std/__mdspan/concepts.h @@ -38,16 +38,15 @@ #include #include #include -#include +#include #include -#include #include #include #include #include #include -#include -#include +#include +#include #include #include @@ -130,6 +129,33 @@ _CCCL_CONCEPT __index_pair_like = _CCCL_REQUIRES_EXPR((_Tp, _IndexType))( template _CCCL_CONCEPT __index_like = is_signed_v<_Tp> || is_unsigned_v<_Tp> || __integral_constant_like<_Tp>; +#if _CCCL_HAS_CONCEPTS() + +template +_CCCL_CONCEPT __has_detect_invalidity = requires(_AccessorPolicy __ap) { + __ap.__has_detect_invalidity( + ::cuda::std::declval(), ::cuda::std::declval()); +}; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +struct __has_detect_invalidity_s : ::cuda::std::false_type +{}; + +template +struct __has_detect_invalidity_s< + _AccessorPolicy, + ::cuda::std::void_t().__has_detect_invalidity( + ::cuda::std::declval(), ::cuda::std::declval()))>> + : ::cuda::std::true_type +{}; + +template +inline constexpr bool __has_detect_invalidity = __has_detect_invalidity_s<_AccessorPolicy>::value; + +#endif // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/include/cuda/std/__mdspan/mdspan.h b/libcudacxx/include/cuda/std/__mdspan/mdspan.h index 5bf6c85f8ae..f8f1811d124 100644 --- a/libcudacxx/include/cuda/std/__mdspan/mdspan.h +++ b/libcudacxx/include/cuda/std/__mdspan/mdspan.h @@ -165,7 +165,14 @@ class mdspan is_nothrow_default_constructible_v && is_nothrow_default_constructible_v && is_nothrow_default_constructible_v) : __base() - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_HIDE_FROM_ABI constexpr mdspan(const mdspan&) = default; _CCCL_HIDE_FROM_ABI constexpr mdspan(mdspan&&) = default; @@ -174,52 +181,106 @@ class mdspan _CCCL_REQUIRES(__constraints::template __can_construct_from_handle_and_variadic<_OtherIndexTypes...>) _CCCL_API explicit constexpr mdspan(data_handle_type __p, _OtherIndexTypes... __exts) : __base(::cuda::std::move(__p), extents_type(static_cast(::cuda::std::move(__exts))...)) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _OtherIndexType, size_t _Size) _CCCL_REQUIRES(__mdspan_detail::__matches_dynamic_rank _CCCL_AND __constraints::template __is_constructible_from_index_type<_OtherIndexType>) _CCCL_API constexpr mdspan(data_handle_type __p, const array<_OtherIndexType, _Size>& __exts) : __base(::cuda::std::move(__p), extents_type{__exts}) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _OtherIndexType, size_t _Size) _CCCL_REQUIRES(__mdspan_detail::__matches_static_rank _CCCL_AND __constraints::template __is_constructible_from_index_type<_OtherIndexType>) _CCCL_API explicit constexpr mdspan(data_handle_type __p, const array<_OtherIndexType, _Size>& __exts) : __base(::cuda::std::move(__p), extents_type{__exts}) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _OtherIndexType, size_t _Size) _CCCL_REQUIRES(__mdspan_detail::__matches_dynamic_rank _CCCL_AND __constraints::template __is_constructible_from_index_type<_OtherIndexType>) _CCCL_API constexpr mdspan(data_handle_type __p, span<_OtherIndexType, _Size> __exts) : __base(::cuda::std::move(__p), extents_type{__exts}) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), mapping().required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _OtherIndexType, size_t _Size) _CCCL_REQUIRES(__mdspan_detail::__matches_static_rank _CCCL_AND __constraints::template __is_constructible_from_index_type<_OtherIndexType>) _CCCL_API explicit constexpr mdspan(data_handle_type __p, span<_OtherIndexType, _Size> __exts) : __base(::cuda::std::move(__p), extents_type{__exts}) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), mapping().required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _AccessorPolicy2 = _AccessorPolicy, class _Mapping2 = mapping_type) _CCCL_REQUIRES( is_default_constructible_v<_AccessorPolicy2> _CCCL_AND is_constructible_v<_Mapping2, const extents_type&>) _CCCL_API constexpr mdspan(data_handle_type __p, const extents_type& __exts) : __base(::cuda::std::move(__p), __exts) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _AccessorPolicy2 = _AccessorPolicy) _CCCL_REQUIRES(is_default_constructible_v<_AccessorPolicy2>) _CCCL_API constexpr mdspan(data_handle_type __p, const mapping_type& __m) : __base(::cuda::std::move(__p), __m) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_API constexpr mdspan(data_handle_type __p, const mapping_type& __m, const accessor_type& __a) : __base(::cuda::std::move(__p), __m, __a) - {} + { + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } + } _CCCL_TEMPLATE(class _OtherElementType, class _OtherExtents, class _OtherLayoutPolicy, class _OtherAccessor) _CCCL_REQUIRES( @@ -233,7 +294,12 @@ class mdspan "mdspan: incompatible data_handle_type for mdspan construction"); static_assert(is_constructible_v, "mdspan: incompatible extents for mdspan construction"); - + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } if constexpr (extents_type::rank() != 0) { // The following precondition is part of the standard, but is unlikely to be triggered. @@ -265,7 +331,12 @@ class mdspan "mdspan: incompatible data_handle_type for mdspan construction"); static_assert(is_constructible_v, "mdspan: incompatible extents for mdspan construction"); - + if constexpr (::cuda::std::__has_detect_invalidity) + { + const auto __tmp = mapping(); // workaround for clang with nodiscard + _CCCL_ASSERT(!accessor().__has_detect_invalidity(data_handle(), __tmp.required_span_size()), + "mdspan: invalid data handle"); + } if constexpr (extents_type::rank() != 0) { // The following precondition is part of the standard, but is unlikely to be triggered. @@ -448,22 +519,25 @@ class mdspan [[nodiscard]] _CCCL_API constexpr bool is_unique() const noexcept(noexcept(::cuda::std::declval().is_unique())) { - return mapping().is_unique(); + const auto __tmp = mapping(); // workaround for clang with nodiscard + return __tmp.is_unique(); } [[nodiscard]] _CCCL_API constexpr bool is_exhaustive() const noexcept(noexcept(::cuda::std::declval().is_exhaustive())) { - auto __tmp = mapping(); // workaround for clang with nodiscard + const auto __tmp = mapping(); // workaround for clang with nodiscard return __tmp.is_exhaustive(); } [[nodiscard]] _CCCL_API constexpr bool is_strided() const noexcept(noexcept(::cuda::std::declval().is_strided())) { - return mapping().is_strided(); + const auto __tmp = mapping(); // workaround for clang with nodiscard + return __tmp.is_strided(); } [[nodiscard]] _CCCL_API constexpr index_type stride(rank_type __r) const { - return mapping().stride(__r); + const auto __tmp = mapping(); // workaround for clang with nodiscard + return __tmp.stride(__r); } };