//@HEADER // ************************************************************************ // // Kokkos v. 4.0 // Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. // See https://kokkos.org/LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE #include static_assert(false, "Including non-public Kokkos header files is not allowed."); #endif #ifndef KOKKOS_VIEW_HPP #define KOKKOS_VIEW_HPP #include #include #include #include #include #include #include #include #include #include #include #ifdef KOKKOS_ENABLE_IMPL_MDSPAN #include #include #include #endif #include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { namespace Impl { template struct ViewArrayAnalysis; template ::non_const_value_type> struct ViewDataAnalysis; template class ViewMapping { public: enum : bool { is_assignable_data_type = false }; enum : bool { is_assignable = false }; }; template constexpr KOKKOS_INLINE_FUNCTION std::size_t count_valid_integers( const IntType i0, const IntType i1, const IntType i2, const IntType i3, const IntType i4, const IntType i5, const IntType i6, const IntType i7) { static_assert(std::is_integral::value, "count_valid_integers() must have integer arguments."); return (i0 != KOKKOS_INVALID_INDEX) + (i1 != KOKKOS_INVALID_INDEX) + (i2 != KOKKOS_INVALID_INDEX) + (i3 != KOKKOS_INVALID_INDEX) + (i4 != KOKKOS_INVALID_INDEX) + (i5 != KOKKOS_INVALID_INDEX) + (i6 != KOKKOS_INVALID_INDEX) + (i7 != KOKKOS_INVALID_INDEX); } // FIXME Ideally, we would not instantiate this function for every possible View // type. We should be able to only pass "extent" when we use mdspan. template KOKKOS_INLINE_FUNCTION void runtime_check_rank( const View&, const bool is_void_spec, const size_t i0, const size_t i1, const size_t i2, const size_t i3, const size_t i4, const size_t i5, const size_t i6, const size_t i7, const char* label) { (void)(label); if (is_void_spec) { const size_t num_passed_args = count_valid_integers(i0, i1, i2, i3, i4, i5, i6, i7); // We either allow to pass as many extents as the dynamic rank is, or // as many extents as the total rank is. In the latter case, the given // extents for the static dimensions must match the // compile-time extents. constexpr int rank = View::rank(); constexpr int dyn_rank = View::rank_dynamic(); const bool n_args_is_dyn_rank = num_passed_args == dyn_rank; const bool n_args_is_rank = num_passed_args == rank; if constexpr (rank != dyn_rank) { if (n_args_is_rank) { size_t new_extents[8] = {i0, i1, i2, i3, i4, i5, i6, i7}; for (int i = dyn_rank; i < rank; ++i) if (new_extents[i] != View::static_extent(i)) { KOKKOS_IF_ON_HOST( const std::string message = "The specified run-time extent for Kokkos::View '" + std::string(label) + "' does not match the compile-time extent in dimension " + std::to_string(i) + ". The given extent is " + std::to_string(new_extents[i]) + " but should be " + std::to_string(View::static_extent(i)) + ".\n"; Kokkos::abort(message.c_str());) KOKKOS_IF_ON_DEVICE( Kokkos::abort( "The specified run-time extents for a Kokkos::View " "do not match the compile-time extents.");) } } } if (!n_args_is_dyn_rank && !n_args_is_rank) { KOKKOS_IF_ON_HOST( const std::string message = "Constructor for Kokkos::View '" + std::string(label) + "' has mismatched number of arguments. The number " "of arguments = " + std::to_string(num_passed_args) + " neither matches the dynamic rank = " + std::to_string(dyn_rank) + " nor the total rank = " + std::to_string(rank) + "\n"; Kokkos::abort(message.c_str());) KOKKOS_IF_ON_DEVICE(Kokkos::abort("Constructor for Kokkos View has " "mismatched number of arguments.");) } } } } /* namespace Impl */ } /* namespace Kokkos */ // Class to provide a uniform type namespace Kokkos { namespace Impl { template struct ViewUniformType; } } // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { /** \class ViewTraits * \brief Traits class for accessing attributes of a View. * * This is an implementation detail of View. It is only of interest * to developers implementing a new specialization of View. * * Template argument options: * - View< DataType > * - View< DataType , Space > * - View< DataType , Space , MemoryTraits > * - View< DataType , ArrayLayout > * - View< DataType , ArrayLayout , Space > * - View< DataType , ArrayLayout , MemoryTraits > * - View< DataType , ArrayLayout , Space , MemoryTraits > * - View< DataType , MemoryTraits > */ template struct ViewTraits; template <> struct ViewTraits { using execution_space = void; using memory_space = void; using HostMirrorSpace = void; using array_layout = void; using memory_traits = void; using specialize = void; using hooks_policy = void; }; template struct ViewTraits { // Ignore an extraneous 'void' using execution_space = typename ViewTraits::execution_space; using memory_space = typename ViewTraits::memory_space; using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; using array_layout = typename ViewTraits::array_layout; using memory_traits = typename ViewTraits::memory_traits; using specialize = typename ViewTraits::specialize; using hooks_policy = typename ViewTraits::hooks_policy; }; template struct ViewTraits< std::enable_if_t::value>, HooksPolicy, Prop...> { using execution_space = typename ViewTraits::execution_space; using memory_space = typename ViewTraits::memory_space; using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; using array_layout = typename ViewTraits::array_layout; using memory_traits = typename ViewTraits::memory_traits; using specialize = typename ViewTraits::specialize; using hooks_policy = HooksPolicy; }; template struct ViewTraits::value>, ArrayLayout, Prop...> { // Specify layout, keep subsequent space and memory traits arguments using execution_space = typename ViewTraits::execution_space; using memory_space = typename ViewTraits::memory_space; using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; using array_layout = ArrayLayout; using memory_traits = typename ViewTraits::memory_traits; using specialize = typename ViewTraits::specialize; using hooks_policy = typename ViewTraits::hooks_policy; }; template struct ViewTraits::value>, Space, Prop...> { // Specify Space, memory traits should be the only subsequent argument. static_assert( std::is_same::execution_space, void>::value && std::is_same::memory_space, void>::value && std::is_same::HostMirrorSpace, void>::value && std::is_same::array_layout, void>::value, "Only one View Execution or Memory Space template argument"); using execution_space = typename Space::execution_space; using memory_space = typename Space::memory_space; using HostMirrorSpace = typename Kokkos::Impl::HostMirror::Space::memory_space; using array_layout = typename execution_space::array_layout; using memory_traits = typename ViewTraits::memory_traits; using specialize = typename ViewTraits::specialize; using hooks_policy = typename ViewTraits::hooks_policy; }; template struct ViewTraits< std::enable_if_t::value>, MemoryTraits, Prop...> { // Specify memory trait, should not be any subsequent arguments static_assert( std::is_same::execution_space, void>::value && std::is_same::memory_space, void>::value && std::is_same::array_layout, void>::value && std::is_same::memory_traits, void>::value && std::is_same::hooks_policy, void>::value, "MemoryTrait is the final optional template argument for a View"); using execution_space = void; using memory_space = void; using HostMirrorSpace = void; using array_layout = void; using memory_traits = MemoryTraits; using specialize = void; using hooks_policy = void; }; template struct ViewTraits { private: // Unpack the properties arguments using prop = ViewTraits; using ExecutionSpace = std::conditional_t::value, typename prop::execution_space, Kokkos::DefaultExecutionSpace>; using MemorySpace = std::conditional_t::value, typename prop::memory_space, typename ExecutionSpace::memory_space>; using ArrayLayout = std::conditional_t::value, typename prop::array_layout, typename ExecutionSpace::array_layout>; using HostMirrorSpace = std::conditional_t< !std::is_void::value, typename prop::HostMirrorSpace, typename Kokkos::Impl::HostMirror::Space>; using MemoryTraits = std::conditional_t::value, typename prop::memory_traits, typename Kokkos::MemoryManaged>; using HooksPolicy = std::conditional_t::value, typename prop::hooks_policy, Kokkos::Experimental::DefaultViewHooks>; // Analyze data type's properties, // May be specialized based upon the layout and value type using data_analysis = Kokkos::Impl::ViewDataAnalysis; public: //------------------------------------ // Data type traits: using data_type = typename data_analysis::type; using const_data_type = typename data_analysis::const_type; using non_const_data_type = typename data_analysis::non_const_type; //------------------------------------ // Compatible array of trivial type traits: using scalar_array_type = typename data_analysis::scalar_array_type; using const_scalar_array_type = typename data_analysis::const_scalar_array_type; using non_const_scalar_array_type = typename data_analysis::non_const_scalar_array_type; //------------------------------------ // Value type traits: using value_type = typename data_analysis::value_type; using const_value_type = typename data_analysis::const_value_type; using non_const_value_type = typename data_analysis::non_const_value_type; //------------------------------------ // Mapping traits: using array_layout = ArrayLayout; using dimension = typename data_analysis::dimension; using specialize = std::conditional_t< std::is_void::value, typename prop::specialize, typename data_analysis::specialize>; /* mapping specialization tag */ static constexpr unsigned rank = dimension::rank; static constexpr unsigned rank_dynamic = dimension::rank_dynamic; //------------------------------------ // Execution space, memory space, memory access traits, and host mirror space. using execution_space = ExecutionSpace; using memory_space = MemorySpace; using device_type = Kokkos::Device; using memory_traits = MemoryTraits; using host_mirror_space = HostMirrorSpace; using hooks_policy = HooksPolicy; using size_type = typename MemorySpace::size_type; enum { is_hostspace = std::is_same::value }; enum { is_managed = MemoryTraits::is_unmanaged == 0 }; enum { is_random_access = MemoryTraits::is_random_access == 1 }; //------------------------------------ }; #ifdef KOKKOS_ENABLE_IMPL_MDSPAN namespace Impl { struct UnsupportedKokkosArrayLayout; template struct MDSpanViewTraits { using mdspan_type = UnsupportedKokkosArrayLayout; }; // "Natural" mdspan for a view if the View's ArrayLayout is supported. template struct MDSpanViewTraits::type>> { using index_type = std::size_t; using extents_type = typename Impl::ExtentsFromDataType::type; using mdspan_layout_type = typename Impl::LayoutFromArrayLayout::type; using accessor_type = Impl::SpaceAwareAccessor< typename Traits::memory_space, Kokkos::default_accessor>; using mdspan_type = mdspan; }; } // namespace Impl #endif // KOKKOS_ENABLE_IMPL_MDSPAN /** \class View * \brief View to an array of data. * * A View represents an array of one or more dimensions. * For details, please refer to Kokkos' tutorial materials. * * \section Kokkos_View_TemplateParameters Template parameters * * This class has both required and optional template parameters. The * \c DataType parameter must always be provided, and must always be * first. The parameters \c Arg1Type, \c Arg2Type, and \c Arg3Type are * placeholders for different template parameters. The default value * of the fifth template parameter \c Specialize suffices for most use * cases. When explaining the template parameters, we won't refer to * \c Arg1Type, \c Arg2Type, and \c Arg3Type; instead, we will refer * to the valid categories of template parameters, in whatever order * they may occur. * * Valid ways in which template arguments may be specified: * - View< DataType > * - View< DataType , Layout > * - View< DataType , Layout , Space > * - View< DataType , Layout , Space , MemoryTraits > * - View< DataType , Space > * - View< DataType , Space , MemoryTraits > * - View< DataType , MemoryTraits > * * \tparam DataType (required) This indicates both the type of each * entry of the array, and the combination of compile-time and * run-time array dimension(s). For example, double* * indicates a one-dimensional array of \c double with run-time * dimension, and int*[3] a two-dimensional array of \c int * with run-time first dimension and compile-time second dimension * (of 3). In general, the run-time dimensions (if any) must go * first, followed by zero or more compile-time dimensions. For * more examples, please refer to the tutorial materials. * * \tparam Space (required) The memory space. * * \tparam Layout (optional) The array's layout in memory. For * example, LayoutLeft indicates a column-major (Fortran style) * layout, and LayoutRight a row-major (C style) layout. If not * specified, this defaults to the preferred layout for the * Space. * * \tparam MemoryTraits (optional) Assertion of the user's intended * access behavior. For example, RandomAccess indicates read-only * access with limited spatial locality, and Unmanaged lets users * wrap externally allocated memory in a View without automatic * deallocation. * * \section Kokkos_View_MT MemoryTraits discussion * * \subsection Kokkos_View_MT_Interp MemoryTraits interpretation depends on * Space * * Some \c MemoryTraits options may have different interpretations for * different \c Space types. For example, with the Cuda device, * \c RandomAccess tells Kokkos to fetch the data through the texture * cache, whereas the non-GPU devices have no such hardware construct. * * \subsection Kokkos_View_MT_PrefUse Preferred use of MemoryTraits * * Users should defer applying the optional \c MemoryTraits parameter * until the point at which they actually plan to rely on it in a * computational kernel. This minimizes the number of template * parameters exposed in their code, which reduces the cost of * compilation. Users may always assign a View without specified * \c MemoryTraits to a compatible View with that specification. * For example: * \code * // Pass in the simplest types of View possible. * void * doSomething (View out, * View in) * { * // Assign the "generic" View in to a RandomAccess View in_rr. * // Note that RandomAccess View objects must have const data. * View in_rr = in; * // ... do something with in_rr and out ... * } * \endcode */ } // namespace Kokkos namespace Kokkos { template struct is_always_assignable_impl; template struct is_always_assignable_impl, Kokkos::View> { using mapping_type = Kokkos::Impl::ViewMapping< typename Kokkos::View::traits, typename Kokkos::View::traits, typename Kokkos::View::traits::specialize>; constexpr static bool value = mapping_type::is_assignable && static_cast(Kokkos::View::rank_dynamic) >= static_cast(Kokkos::View::rank_dynamic); }; template using is_always_assignable = is_always_assignable_impl< std::remove_reference_t, std::remove_const_t>>; template inline constexpr bool is_always_assignable_v = is_always_assignable::value; template constexpr bool is_assignable(const Kokkos::View& dst, const Kokkos::View& src) { using DstTraits = typename Kokkos::View::traits; using SrcTraits = typename Kokkos::View::traits; using mapping_type = Kokkos::Impl::ViewMapping; return is_always_assignable_v, Kokkos::View> || (mapping_type::is_assignable && ((DstTraits::dimension::rank_dynamic >= 1) || (dst.static_extent(0) == src.extent(0))) && ((DstTraits::dimension::rank_dynamic >= 2) || (dst.static_extent(1) == src.extent(1))) && ((DstTraits::dimension::rank_dynamic >= 3) || (dst.static_extent(2) == src.extent(2))) && ((DstTraits::dimension::rank_dynamic >= 4) || (dst.static_extent(3) == src.extent(3))) && ((DstTraits::dimension::rank_dynamic >= 5) || (dst.static_extent(4) == src.extent(4))) && ((DstTraits::dimension::rank_dynamic >= 6) || (dst.static_extent(5) == src.extent(5))) && ((DstTraits::dimension::rank_dynamic >= 7) || (dst.static_extent(6) == src.extent(6))) && ((DstTraits::dimension::rank_dynamic >= 8) || (dst.static_extent(7) == src.extent(7)))); } } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- #include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { // FIXME_OPENMPTARGET - The `declare target` is needed for the Intel GPUs with // the OpenMPTarget backend #if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) #pragma omp declare target #endif inline constexpr Kokkos::ALL_t ALL{}; #if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) #pragma omp end declare target #endif inline constexpr Kokkos::Impl::WithoutInitializing_t WithoutInitializing{}; inline constexpr Kokkos::Impl::AllowPadding_t AllowPadding{}; /** \brief Create View allocation parameter bundle from argument list. * * Valid argument list members are: * 1) label as a "string" or std::string * 2) memory space instance of the View::memory_space type * 3) execution space instance compatible with the View::memory_space * 4) Kokkos::WithoutInitializing to bypass initialization * 4) Kokkos::AllowPadding to allow allocation to pad dimensions for memory * alignment */ template inline Impl::ViewCtorProp::type...> view_alloc(Args const&... args) { using return_type = Impl::ViewCtorProp::type...>; static_assert(!return_type::has_pointer, "Cannot give pointer-to-memory for view allocation"); return return_type(args...); } template KOKKOS_INLINE_FUNCTION Impl::ViewCtorProp::type...> view_wrap(Args const&... args) { using return_type = Impl::ViewCtorProp::type...>; static_assert(!return_type::has_memory_space && !return_type::has_execution_space && !return_type::has_label && return_type::has_pointer, "Must only give pointer-to-memory for view wrapping"); return return_type(args...); } } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { template class View; template struct is_view : public std::false_type {}; template struct is_view> : public std::true_type {}; template struct is_view> : public std::true_type {}; template inline constexpr bool is_view_v = is_view::value; template class View : public ViewTraits { private: template friend class View; template friend class Kokkos::Impl::ViewMapping; using view_tracker_type = Kokkos::Impl::ViewTracker; public: using traits = ViewTraits; private: using map_type = Kokkos::Impl::ViewMapping; template friend struct Kokkos::Impl::ViewTracker; using hooks_policy = typename traits::hooks_policy; view_tracker_type m_track; map_type m_map; public: //---------------------------------------- /** \brief Compatible view of array of scalar types */ using array_type = View; /** \brief Compatible view of const data type */ using const_type = View; /** \brief Compatible view of non-const data type */ using non_const_type = View; /** \brief Compatible HostMirror view */ using HostMirror = View, typename traits::hooks_policy>; /** \brief Compatible HostMirror view */ using host_mirror_type = View; /** \brief Unified types */ using uniform_type = typename Impl::ViewUniformType::type; using uniform_const_type = typename Impl::ViewUniformType::const_type; using uniform_runtime_type = typename Impl::ViewUniformType::runtime_type; using uniform_runtime_const_type = typename Impl::ViewUniformType::runtime_const_type; using uniform_nomemspace_type = typename Impl::ViewUniformType::nomemspace_type; using uniform_const_nomemspace_type = typename Impl::ViewUniformType::const_nomemspace_type; using uniform_runtime_nomemspace_type = typename Impl::ViewUniformType::runtime_nomemspace_type; using uniform_runtime_const_nomemspace_type = typename Impl::ViewUniformType::runtime_const_nomemspace_type; //---------------------------------------- // Domain rank and extents static constexpr Impl::integral_constant rank = {}; static constexpr Impl::integral_constant rank_dynamic = {}; #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 enum {Rank KOKKOS_DEPRECATED_WITH_COMMENT("Use rank instead.") = map_type::Rank}; #endif template KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< std::is_integral::value, size_t> extent(const iType& r) const noexcept { return m_map.extent(r); } static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent( const unsigned r) noexcept { return map_type::static_extent(r); } template KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< std::is_integral::value, int> extent_int(const iType& r) const noexcept { return static_cast(m_map.extent(r)); } KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() const { return m_map.layout(); } //---------------------------------------- /* Deprecate all 'dimension' functions in favor of * ISO/C++ vocabulary 'extent'. */ KOKKOS_INLINE_FUNCTION constexpr size_t size() const { return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * m_map.dimension_6() * m_map.dimension_7(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } template KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< std::is_integral::value, size_t> stride(iType r) const { return ( r == 0 ? m_map.stride_0() : (r == 1 ? m_map.stride_1() : (r == 2 ? m_map.stride_2() : (r == 3 ? m_map.stride_3() : (r == 4 ? m_map.stride_4() : (r == 5 ? m_map.stride_5() : (r == 6 ? m_map.stride_6() : m_map.stride_7()))))))); } template KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { m_map.stride(s); } //---------------------------------------- // Range span is the span which contains all members. using reference_type = typename map_type::reference_type; using pointer_type = typename map_type::pointer_type; enum { reference_type_is_lvalue_reference = std::is_lvalue_reference::value }; KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { return m_map.span_is_contiguous(); } KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { return m_map.data() != nullptr; } KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return m_map.data(); } //---------------------------------------- // Allow specializations to query their specialized map KOKKOS_INLINE_FUNCTION const Kokkos::Impl::ViewMapping& impl_map() const { return m_map; } KOKKOS_INLINE_FUNCTION const Kokkos::Impl::SharedAllocationTracker& impl_track() const { return m_track.m_tracker; } //---------------------------------------- private: static constexpr bool is_layout_left = std::is_same::value; static constexpr bool is_layout_right = std::is_same::value; static constexpr bool is_layout_stride = std::is_same::value; static constexpr bool is_default_map = std::is_void::value && (is_layout_left || is_layout_right || is_layout_stride); #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) #define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::View ERROR: attempt to access inaccessible memory space", \ __VA_ARGS__); \ Kokkos::Impl::view_verify_operator_bounds( \ __VA_ARGS__); #else #define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::View ERROR: attempt to access inaccessible memory space", \ __VA_ARGS__); #endif template static KOKKOS_FUNCTION void check_access_member_function_valid_args(Is...) { static_assert(rank <= sizeof...(Is)); static_assert(sizeof...(Is) <= 8); static_assert(Kokkos::Impl::are_integral::value); } template static KOKKOS_FUNCTION void check_operator_parens_valid_args(Is...) { static_assert(rank == sizeof...(Is)); static_assert(Kokkos::Impl::are_integral::value); } public: //------------------------------ // Rank 1 default map operator() template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && // (1 == rank) && is_default_map && !is_layout_stride), reference_type> operator()(I0 i0) const { check_operator_parens_valid_args(i0); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) return m_map.m_impl_handle[i0]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && // (1 == rank) && is_default_map && is_layout_stride), reference_type> operator()(I0 i0) const { check_operator_parens_valid_args(i0); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; } //------------------------------ // Rank 1 operator[] template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< ((1 == rank) && Kokkos::Impl::are_integral::value && !is_default_map), reference_type> operator[](I0 i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) return m_map.reference(i0); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && is_default_map && !is_layout_stride), reference_type> operator[](I0 i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) return m_map.m_impl_handle[i0]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && is_default_map && is_layout_stride), reference_type> operator[](I0 i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; } //------------------------------ // Rank 2 default map operator() template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && // (2 == rank) && is_default_map && (is_layout_left || is_layout_right || is_layout_stride)), reference_type> operator()(I0 i0, I1 i1) const { check_operator_parens_valid_args(i0, i1); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1) if constexpr (is_layout_left) { if constexpr (rank_dynamic == 0) return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; else return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; } else if constexpr (is_layout_right) { if constexpr (rank_dynamic == 0) return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; else return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; } else { static_assert(is_layout_stride); return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + i1 * m_map.m_impl_offset.m_stride.S1]; } #if defined KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } // Rank 0 -> 8 operator() except for rank-1 and rank-2 with default map which // have "inlined" versions above template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && // (2 != rank) && (1 != rank) && (0 != rank) && is_default_map), reference_type> operator()(Is... indices) const { check_operator_parens_valid_args(indices...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) return m_map.m_impl_handle[m_map.m_impl_offset(indices...)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && // ((0 == rank) || !is_default_map)), reference_type> operator()(Is... indices) const { check_operator_parens_valid_args(indices...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) return m_map.reference(indices...); } //------------------------------ // Rank 0 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (0 == rank)), reference_type> access(Is... extra) const { check_access_member_function_valid_args(extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, extra...) return m_map.reference(); } //------------------------------ // Rank 1 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (1 == rank) && !is_default_map), reference_type> access(I0 i0, Is... extra) const { check_access_member_function_valid_args(i0, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) return m_map.reference(i0); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (1 == rank) && is_default_map && !is_layout_stride), reference_type> access(I0 i0, Is... extra) const { check_access_member_function_valid_args(i0, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) return m_map.m_impl_handle[i0]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (1 == rank) && is_default_map && is_layout_stride), reference_type> access(I0 i0, Is... extra) const { check_access_member_function_valid_args(i0, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; } //------------------------------ // Rank 2 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (2 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, Is... extra) const { check_access_member_function_valid_args(i0, i1, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) return m_map.reference(i0, i1); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (2 == rank) && is_default_map && (is_layout_left || is_layout_right || is_layout_stride)), reference_type> access(I0 i0, I1 i1, Is... extra) const { check_access_member_function_valid_args(i0, i1, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) if constexpr (is_layout_left) { if constexpr (rank_dynamic == 0) return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; else return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; } else if constexpr (is_layout_right) { if constexpr (rank_dynamic == 0) return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; else return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; } else { static_assert(is_layout_stride); return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + i1 * m_map.m_impl_offset.m_stride.S1]; } #if defined KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } //------------------------------ // Rank 3 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (3 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (3 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) return m_map.reference(i0, i1, i2); } //------------------------------ // Rank 4 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (4 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (4 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) return m_map.reference(i0, i1, i2, i3); } //------------------------------ // Rank 5 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (5 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, extra...) return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (5 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, extra...) return m_map.reference(i0, i1, i2, i3, i4); } //------------------------------ // Rank 6 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (6 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, extra...) return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (6 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, extra...) return m_map.reference(i0, i1, i2, i3, i4, i5); } //------------------------------ // Rank 7 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (7 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, extra...) return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::always_true::value && (7 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, extra...) return m_map.reference(i0, i1, i2, i3, i4, i5, i6); } //------------------------------ // Rank 8 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (8 == rank) && is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, i7, extra...) return m_map .m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::always_true::value && (8 == rank) && !is_default_map), reference_type> access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, Is... extra) const { check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, extra...); KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, i7, extra...) return m_map.reference(i0, i1, i2, i3, i4, i5, i6, i7); } #undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY //---------------------------------------- // Standard destructor, constructors, and assignment operators KOKKOS_DEFAULTED_FUNCTION ~View() = default; KOKKOS_DEFAULTED_FUNCTION View() = default; KOKKOS_FUNCTION View(const View& other) : m_track(other.m_track), m_map(other.m_map) { KOKKOS_IF_ON_HOST((hooks_policy::copy_construct(*this, other);)) } KOKKOS_FUNCTION View(View&& other) : m_track{std::move(other.m_track)}, m_map{std::move(other.m_map)} { KOKKOS_IF_ON_HOST((hooks_policy::move_construct(*this, other);)) } KOKKOS_FUNCTION View& operator=(const View& other) { m_map = other.m_map; m_track = other.m_track; KOKKOS_IF_ON_HOST((hooks_policy::copy_assign(*this, other);)) return *this; } KOKKOS_FUNCTION View& operator=(View&& other) { m_map = std::move(other.m_map); m_track = std::move(other.m_track); KOKKOS_IF_ON_HOST((hooks_policy::move_assign(*this, other);)) return *this; } //---------------------------------------- // Compatible view copy constructor and assignment // may assign unmanaged from managed. template KOKKOS_INLINE_FUNCTION View( const View& rhs, std::enable_if_t::traits, typename traits::specialize>::is_assignable_data_type>* = nullptr) : m_track(rhs), m_map() { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View copy construction"); Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); } template KOKKOS_INLINE_FUNCTION std::enable_if_t< Kokkos::Impl::ViewMapping< traits, typename View::traits, typename traits::specialize>::is_assignable_data_type, View>& operator=(const View& rhs) { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View copy assignment"); Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); m_track.assign(rhs); return *this; } //---------------------------------------- // Compatible subview constructor // may assign unmanaged from managed. template KOKKOS_INLINE_FUNCTION View(const View& src_view, const Arg0 arg0, Args... args) : m_track(src_view), m_map() { using SrcType = View; using Mapping = Kokkos::Impl::ViewMapping; using DstType = typename Mapping::type; static_assert( Kokkos::Impl::ViewMapping::is_assignable, "Subview construction requires compatible view and subview arguments"); Mapping::assign(m_map, src_view.m_map, arg0, args...); } //---------------------------------------- // Allocation tracking properties KOKKOS_INLINE_FUNCTION int use_count() const { return m_track.m_tracker.use_count(); } inline const std::string label() const { return m_track.m_tracker .template get_label(); } public: //---------------------------------------- // Allocation according to allocation properties and array layout template explicit inline View( const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout) : m_track(), m_map() { // Copy the input allocation properties with possibly defaulted properties // We need to split it in two to avoid MSVC compiler errors auto prop_copy_tmp = Impl::with_properties_if_unset(arg_prop, std::string{}); auto prop_copy = Impl::with_properties_if_unset( prop_copy_tmp, typename traits::device_type::memory_space{}, typename traits::device_type::execution_space{}); using alloc_prop = decltype(prop_copy); static_assert(traits::is_managed, "View allocation constructor requires managed memory"); if (alloc_prop::initialize && !alloc_prop::execution_space::impl_is_initialized()) { // If initializing view data then // the execution space must be initialized. Kokkos::Impl::throw_runtime_exception( "Constructing View and initializing data with uninitialized " "execution space"); } #ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { size_t i0 = arg_layout.dimension[0]; size_t i1 = arg_layout.dimension[1]; size_t i2 = arg_layout.dimension[2]; size_t i3 = arg_layout.dimension[3]; size_t i4 = arg_layout.dimension[4]; size_t i5 = arg_layout.dimension[5]; size_t i6 = arg_layout.dimension[6]; size_t i7 = arg_layout.dimension[7]; const std::string& alloc_name = Impl::get_property(prop_copy); Impl::runtime_check_rank( *this, std::is_same::value, i0, i1, i2, i3, i4, i5, i6, i7, alloc_name.c_str()); } #endif Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( prop_copy, arg_layout, Impl::ViewCtorProp::has_execution_space); // Setup and initialization complete, start tracking m_track.m_tracker.assign_allocated_record_to_uninitialized(record); } KOKKOS_INLINE_FUNCTION void assign_data(pointer_type arg_data) { m_track.m_tracker.clear(); m_map.assign_data(arg_data); } // Wrap memory according to properties and array layout template explicit KOKKOS_INLINE_FUNCTION View( const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout) : m_track() // No memory tracking , m_map(arg_prop, arg_layout) { static_assert( std::is_same::pointer_type>::value, "Constructing View to wrap user memory must supply matching pointer " "type"); #ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { size_t i0 = arg_layout.dimension[0]; size_t i1 = arg_layout.dimension[1]; size_t i2 = arg_layout.dimension[2]; size_t i3 = arg_layout.dimension[3]; size_t i4 = arg_layout.dimension[4]; size_t i5 = arg_layout.dimension[5]; size_t i6 = arg_layout.dimension[6]; size_t i7 = arg_layout.dimension[7]; Impl::runtime_check_rank( *this, std::is_same::value, i0, i1, i2, i3, i4, i5, i6, i7, "UNMANAGED"); } #endif } // Simple dimension-only layout template explicit inline View( const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, size_t> const arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) : View(arg_prop, typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); } template explicit KOKKOS_INLINE_FUNCTION View( const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, size_t> const arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) : View(arg_prop, typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); } // Allocate with label and layout template explicit inline View( const Label& arg_label, std::enable_if_t::value, typename traits::array_layout> const& arg_layout) : View(Impl::ViewCtorProp(arg_label), arg_layout) {} // Allocate label and layout, must disambiguate from subview constructor. template explicit inline View( const Label& arg_label, std::enable_if_t::value, const size_t> arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) : View(Impl::ViewCtorProp(arg_label), typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); } // Construct view from ViewTracker and map // This should be the preferred method because future extensions may need to // use the ViewTracker class. template KOKKOS_INLINE_FUNCTION View( const view_tracker_type& track, const Kokkos::Impl::ViewMapping& map) : m_track(track), m_map() { using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View copy construction"); Mapping::assign(m_map, map, track.m_tracker); } // Construct View from internal shared allocation tracker object and map // This is here for backwards compatibility for classes that derive from // Kokkos::View template KOKKOS_INLINE_FUNCTION View( const typename view_tracker_type::track_type& track, const Kokkos::Impl::ViewMapping& map) : m_track(track), m_map() { using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View copy construction"); Mapping::assign(m_map, map, track); } //---------------------------------------- // Memory span required to wrap these dimensions. static constexpr size_t required_allocation_size( typename traits::array_layout const& layout) { return map_type::memory_span(layout); } static constexpr size_t required_allocation_size( const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, const size_t arg_N6 = 0, const size_t arg_N7 = 0) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); return map_type::memory_span(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); } explicit KOKKOS_INLINE_FUNCTION View( pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) : View(Impl::ViewCtorProp(arg_ptr), typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); } explicit KOKKOS_INLINE_FUNCTION View( pointer_type arg_ptr, const typename traits::array_layout& arg_layout) : View(Impl::ViewCtorProp(arg_ptr), arg_layout) {} //---------------------------------------- // Shared scratch memory constructor static KOKKOS_INLINE_FUNCTION size_t shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); const size_t num_passed_args = Impl::count_valid_integers( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7); if (std::is_void::value && num_passed_args != rank_dynamic) { Kokkos::abort( "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); } return View::shmem_size(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); } private: // Want to be able to align to minimum scratch alignment or sizeof or alignof // elements static constexpr size_t scratch_value_alignment = max({sizeof(typename traits::value_type), alignof(typename traits::value_type), static_cast( traits::execution_space::scratch_memory_space::ALIGN)}); public: static KOKKOS_INLINE_FUNCTION size_t shmem_size(typename traits::array_layout const& arg_layout) { return map_type::memory_span(arg_layout) + scratch_value_alignment; } explicit KOKKOS_INLINE_FUNCTION View( const typename traits::execution_space::scratch_memory_space& arg_space, const typename traits::array_layout& arg_layout) : View(Impl::ViewCtorProp(reinterpret_cast( arg_space.get_shmem_aligned(map_type::memory_span(arg_layout), scratch_value_alignment))), arg_layout) {} explicit KOKKOS_INLINE_FUNCTION View( const typename traits::execution_space::scratch_memory_space& arg_space, const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) : View(Impl::ViewCtorProp( reinterpret_cast(arg_space.get_shmem_aligned( map_type::memory_span(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)), scratch_value_alignment))), typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) { static_assert(traits::array_layout::is_extent_constructible, "Layout is not constructible from extent arguments. Use " "overload taking a layout object instead."); } //---------------------------------------- // MDSpan converting constructors #ifdef KOKKOS_ENABLE_IMPL_MDSPAN template ::mdspan_type> KOKKOS_INLINE_FUNCTION #if defined(__cpp_conditional_explicit) && \ (__cpp_conditional_explicit >= 201806L) // FIXME C++20 reevaluate after determining minium compiler versions explicit(traits::is_managed) #endif View(const typename Impl::MDSpanViewTraits::mdspan_type& mds, std::enable_if_t< !std::is_same_v>* = nullptr) : View(mds.data_handle(), Impl::array_layout_from_mapping< typename traits::array_layout, typename Impl::MDSpanViewTraits::mdspan_type>( mds.mapping())) { } template KOKKOS_INLINE_FUNCTION #if defined(__cpp_conditional_explicit) && \ (__cpp_conditional_explicit >= 201806L) // FIXME C++20 reevaluate after determining minium compiler versions explicit(!std::is_convertible_v< Kokkos::mdspan, typename Impl::MDSpanViewTraits::mdspan_type>) #endif View(const Kokkos::mdspan& mds) : View(typename Impl::MDSpanViewTraits::mdspan_type(mds)) { } //---------------------------------------- // Conversion to MDSpan template , typename Impl::MDSpanViewTraits::mdspan_type>>> KOKKOS_INLINE_FUNCTION constexpr operator mdspan< OtherElementType, OtherExtents, OtherLayoutPolicy, OtherAccessor>() { using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; return mdspan_type{data(), Impl::mapping_from_view_mapping(m_map)}; } template >, typename = std::enable_if_t>> KOKKOS_INLINE_FUNCTION constexpr auto to_mdspan( const OtherAccessorType& other_accessor = typename Impl::MDSpanViewTraits::accessor_type()) { using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; using ret_mdspan_type = mdspan; return ret_mdspan_type{data(), Impl::mapping_from_view_mapping(m_map), other_accessor}; } #endif // KOKKOS_ENABLE_IMPL_MDSPAN }; template KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const View&) { return View::rank(); } namespace Impl { template struct RankDataType { using type = typename RankDataType::type*; }; template struct RankDataType { using type = ValueType; }; template KOKKOS_FUNCTION std::enable_if_t< N == View::rank() && std::is_same::specialize, void>::value, View> as_view_of_rank_n(View v) { return v; } // Placeholder implementation to compile generic code for DynRankView; should // never be called template KOKKOS_FUNCTION std::enable_if_t< N != View::rank() && std::is_same::specialize, void>::value, View::value_type, N>::type, Args...>> as_view_of_rank_n(View) { Kokkos::abort("Trying to get at a View of the wrong rank"); return {}; } template void apply_to_view_of_static_rank(Function&& f, View a) { f(a); } } // namespace Impl //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Impl { template struct TypeListToViewTraits; template struct TypeListToViewTraits> { using type = ViewTraits; }; // It is not safe to assume that subviews of views with the Aligned memory trait // are also aligned. Hence, just remove that attribute for subviews. template struct RemoveAlignedMemoryTrait { private: using type_list_in = Kokkos::Impl::type_list; using memory_traits = typename ViewTraits::memory_traits; using type_list_in_wo_memory_traits = typename Kokkos::Impl::type_list_remove_first::type; using new_memory_traits = Kokkos::MemoryTraits; using new_type_list = typename Kokkos::Impl::concat_type_list< type_list_in_wo_memory_traits, Kokkos::Impl::type_list>::type; public: using type = typename TypeListToViewTraits::type; }; } // namespace Impl template KOKKOS_INLINE_FUNCTION auto subview(const View& src, Args... args) { static_assert(View::rank == sizeof...(Args), "subview requires one argument for each source View rank"); return typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , typename Impl::RemoveAlignedMemoryTrait::type, Args...>::type(src, args...); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 template KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION auto subview(const View& src, Args... args) { static_assert(View::rank == sizeof...(Args), "subview requires one argument for each source View rank"); static_assert(Kokkos::is_memory_traits::value); return typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , typename Impl::RemoveAlignedMemoryTrait::type, Args...>::type(src, args...); } #endif template using Subview = decltype(subview(std::declval(), std::declval()...)); } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { template KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, const View& rhs) { // Same data, layout, dimensions using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; return std::is_same::value && std::is_same::value && std::is_same::value && View::rank() == View::rank() && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); } template KOKKOS_INLINE_FUNCTION bool operator!=(const View& lhs, const View& rhs) { return !(operator==(lhs, rhs)); } } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { namespace Impl { template struct CommonViewValueType; template struct CommonViewValueType { using value_type = std::common_type_t; }; template struct CommonViewAllocProp; template struct CommonViewAllocProp { using value_type = ValueType; using scalar_array_type = ValueType; template KOKKOS_INLINE_FUNCTION CommonViewAllocProp(const Views&...) {} }; template struct DeduceCommonViewAllocProp; // Base case must provide types for: // 1. specialize 2. value_type 3. is_view 4. prop_type template struct DeduceCommonViewAllocProp { using specialize = typename FirstView::traits::specialize; using value_type = typename FirstView::traits::value_type; enum : bool { is_view = is_view::value }; using prop_type = CommonViewAllocProp; }; template struct DeduceCommonViewAllocProp { using NextTraits = DeduceCommonViewAllocProp; using first_specialize = typename FirstView::traits::specialize; using first_value_type = typename FirstView::traits::value_type; enum : bool { first_is_view = is_view::value }; using next_specialize = typename NextTraits::specialize; using next_value_type = typename NextTraits::value_type; enum : bool { next_is_view = NextTraits::is_view }; // common types // determine specialize type // if first and next specialize differ, but are not the same specialize, error // out static_assert(!(!std::is_same::value && !std::is_void::value && !std::is_void::value), "Kokkos DeduceCommonViewAllocProp ERROR: Only one non-void " "specialize trait allowed"); // otherwise choose non-void specialize if either/both are non-void using specialize = std::conditional_t< std::is_same::value, first_specialize, std::conditional_t<(std::is_void::value && !std::is_void::value), next_specialize, first_specialize>>; using value_type = typename CommonViewValueType::value_type; enum : bool { is_view = (first_is_view && next_is_view) }; using prop_type = CommonViewAllocProp; }; } // end namespace Impl template using DeducedCommonPropsType = typename Impl::DeduceCommonViewAllocProp::prop_type; // This function is required in certain scenarios where users customize // Kokkos View internals. One example are dynamic length embedded ensemble // types. The function is used to propagate necessary information // (like the ensemble size) when creating new views. // However, most of the time it is called with a single view. // Furthermore, the propagated information is not just for view allocations. // From what I can tell, the type of functionality provided by // common_view_alloc_prop is the equivalent of propagating accessors in mdspan, // a mechanism we will eventually use to replace this clunky approach here, when // we are finally mdspan based. // TODO: get rid of this when we have mdspan template KOKKOS_INLINE_FUNCTION DeducedCommonPropsType common_view_alloc_prop( Views const&... views) { return DeducedCommonPropsType(views...); } } // namespace Kokkos #include #include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- #endif /* #ifndef KOKKOS_VIEW_HPP */