//@HEADER // ************************************************************************ // // Kokkos v. 4.0 // Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. // See https://kokkos.org/LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER #ifndef KOKKOS_OFFSETVIEW_HPP_ #define KOKKOS_OFFSETVIEW_HPP_ #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE #define KOKKOS_IMPL_PUBLIC_INCLUDE #define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW #endif #include #include namespace Kokkos { namespace Experimental { //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- template class OffsetView; template struct is_offset_view : public std::false_type {}; template struct is_offset_view> : public std::true_type {}; template struct is_offset_view> : public std::true_type {}; template inline constexpr bool is_offset_view_v = is_offset_view::value; #define KOKKOS_INVALID_OFFSET int64_t(0x7FFFFFFFFFFFFFFFLL) #define KOKKOS_INVALID_INDEX_RANGE \ { KOKKOS_INVALID_OFFSET, KOKKOS_INVALID_OFFSET } template ::value && std::is_signed::value, iType> = 0> using IndexRange = Kokkos::Array; using index_list_type = std::initializer_list; // template ::value && // std::is_signed::value, iType > = 0> using min_index_type = // std::initializer_list; namespace Impl { template struct GetOffsetViewTypeFromViewType { using type = OffsetView; }; template KOKKOS_INLINE_FUNCTION bool offsetview_verify_operator_bounds( const MapType&, const BeginsType&) { return true; } template KOKKOS_INLINE_FUNCTION bool offsetview_verify_operator_bounds( const MapType& map, const BeginsType& begins, const iType& i, Args... args) { const bool legalIndex = (int64_t(i) >= begins[R]) && (int64_t(i) <= int64_t(begins[R] + map.extent(R) - 1)); return legalIndex && offsetview_verify_operator_bounds(map, begins, args...); } template inline void offsetview_error_operator_bounds(char*, int, const MapType&, const BeginsType&) {} template inline void offsetview_error_operator_bounds(char* buf, int len, const MapType& map, const BeginsType begins, const iType& i, Args... args) { const int64_t b = begins[R]; const int64_t e = b + map.extent(R) - 1; const int n = snprintf(buf, len, " %ld <= %ld <= %ld %c", static_cast(b), static_cast(i), static_cast(e), (sizeof...(Args) ? ',' : ')')); offsetview_error_operator_bounds(buf + n, len - n, map, begins, args...); } template KOKKOS_INLINE_FUNCTION void offsetview_verify_operator_bounds( Kokkos::Impl::SharedAllocationTracker const& tracker, const MapType& map, const BeginsType& begins, Args... args) { if (!offsetview_verify_operator_bounds<0>(map, begins, args...)) { KOKKOS_IF_ON_HOST( (enum {LEN = 1024}; char buffer[LEN]; const std::string label = tracker.template get_label(); int n = snprintf(buffer, LEN, "OffsetView bounds error of view labeled %s (", label.c_str()); offsetview_error_operator_bounds<0>(buffer + n, LEN - n, map, begins, args...); Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) KOKKOS_IF_ON_DEVICE( (Kokkos::abort("OffsetView bounds error"); (void)tracker;)) } } inline void runtime_check_rank_host(const size_t rank_dynamic, const size_t rank, const index_list_type minIndices, const std::string& label) { bool isBad = false; std::string message = "Kokkos::Experimental::OffsetView ERROR: for OffsetView labeled '" + label + "':"; if (rank_dynamic != rank) { message += "The full rank must be the same as the dynamic rank. full rank = "; message += std::to_string(rank) + " dynamic rank = " + std::to_string(rank_dynamic) + "\n"; isBad = true; } size_t numOffsets = 0; for (size_t i = 0; i < minIndices.size(); ++i) { if (minIndices.begin()[i] != KOKKOS_INVALID_OFFSET) numOffsets++; } if (numOffsets != rank_dynamic) { message += "The number of offsets provided ( " + std::to_string(numOffsets) + " ) must equal the dynamic rank ( " + std::to_string(rank_dynamic) + " )."; isBad = true; } if (isBad) Kokkos::abort(message.c_str()); } KOKKOS_INLINE_FUNCTION void runtime_check_rank_device(const size_t rank_dynamic, const size_t rank, const index_list_type minIndices) { if (rank_dynamic != rank) { Kokkos::abort( "The full rank of an OffsetView must be the same as the dynamic rank."); } size_t numOffsets = 0; for (size_t i = 0; i < minIndices.size(); ++i) { if (minIndices.begin()[i] != KOKKOS_INVALID_OFFSET) numOffsets++; } if (numOffsets != rank) { Kokkos::abort( "The number of offsets provided to an OffsetView constructor must " "equal the dynamic rank."); } } } // namespace Impl template class OffsetView : public ViewTraits { public: using traits = ViewTraits; private: template friend class OffsetView; template friend class View; // FIXME delete this line template friend class Kokkos::Impl::ViewMapping; using map_type = Kokkos::Impl::ViewMapping; using track_type = Kokkos::Impl::SharedAllocationTracker; public: enum { Rank = map_type::Rank }; using begins_type = Kokkos::Array; template ::value, iType> = 0> KOKKOS_FUNCTION int64_t begin(const iType local_dimension) const { return local_dimension < Rank ? m_begins[local_dimension] : KOKKOS_INVALID_OFFSET; } KOKKOS_FUNCTION begins_type begins() const { return m_begins; } template ::value, iType> = 0> KOKKOS_FUNCTION int64_t end(const iType local_dimension) const { return begin(local_dimension) + m_map.extent(local_dimension); } private: track_type m_track; map_type m_map; begins_type m_begins; public: //---------------------------------------- /** \brief Compatible view of array of scalar types */ using array_type = OffsetView; /** \brief Compatible view of const data type */ using const_type = OffsetView; /** \brief Compatible view of non-const data type */ using non_const_type = OffsetView; /** \brief Compatible HostMirror view */ using HostMirror = OffsetView; //---------------------------------------- // Domain rank and extents /** \brief rank() to be implemented */ // KOKKOS_FUNCTION // static // constexpr unsigned rank() { return map_type::Rank; } template KOKKOS_FUNCTION constexpr std::enable_if_t::value, size_t> extent(const iType& r) const { return m_map.extent(r); } template KOKKOS_FUNCTION constexpr std::enable_if_t::value, int> extent_int(const iType& r) const { return static_cast(m_map.extent(r)); } KOKKOS_FUNCTION constexpr typename traits::array_layout layout() const { return m_map.layout(); } KOKKOS_FUNCTION constexpr size_t size() const { return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * m_map.dimension_6() * m_map.dimension_7(); } KOKKOS_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } KOKKOS_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } KOKKOS_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } KOKKOS_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } KOKKOS_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } KOKKOS_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } KOKKOS_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } KOKKOS_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } template KOKKOS_FUNCTION constexpr std::enable_if_t::value, size_t> stride(iType r) const { return ( r == 0 ? m_map.stride_0() : (r == 1 ? m_map.stride_1() : (r == 2 ? m_map.stride_2() : (r == 3 ? m_map.stride_3() : (r == 4 ? m_map.stride_4() : (r == 5 ? m_map.stride_5() : (r == 6 ? m_map.stride_6() : m_map.stride_7()))))))); } template KOKKOS_FUNCTION void stride(iType* const s) const { m_map.stride(s); } //---------------------------------------- // Range span is the span which contains all members. using reference_type = typename map_type::reference_type; using pointer_type = typename map_type::pointer_type; enum { reference_type_is_lvalue_reference = std::is_lvalue_reference::value }; KOKKOS_FUNCTION constexpr size_t span() const { return m_map.span(); } KOKKOS_FUNCTION bool span_is_contiguous() const { return m_map.span_is_contiguous(); } KOKKOS_FUNCTION constexpr bool is_allocated() const { return m_map.data() != nullptr; } KOKKOS_FUNCTION constexpr pointer_type data() const { return m_map.data(); } //---------------------------------------- // Allow specializations to query their specialized map KOKKOS_FUNCTION const Kokkos::Impl::ViewMapping& implementation_map() const { return m_map; } //---------------------------------------- private: static constexpr bool is_layout_left = std::is_same::value; static constexpr bool is_layout_right = std::is_same::value; static constexpr bool is_layout_stride = std::is_same::value; static constexpr bool is_default_map = std::is_void::value && (is_layout_left || is_layout_right || is_layout_stride); #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) #define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ "space"); \ Kokkos::Experimental::Impl::offsetview_verify_operator_bounds< \ typename traits::memory_space> \ ARG; #else #define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ "space"); #endif public: //------------------------------ // Rank 0 operator() KOKKOS_FORCEINLINE_FUNCTION reference_type operator()() const { return m_map.reference(); } //------------------------------ // Rank 1 operator() template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), reference_type> operator()(const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.reference(j0); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && is_default_map && !is_layout_stride), reference_type> operator()(const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.m_impl_handle[j0]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && is_default_map && is_layout_stride), reference_type> operator()(const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; } //------------------------------ // Rank 1 operator[] template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), reference_type> operator[](const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.reference(j0); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && is_default_map && !is_layout_stride), reference_type> operator[](const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.m_impl_handle[j0]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && is_default_map && is_layout_stride), reference_type> operator[](const I0& i0) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) const size_t j0 = i0 - m_begins[0]; return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; } //------------------------------ // Rank 2 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (2 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; return m_map.reference(j0, j1); } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (2 == Rank) && is_default_map && (is_layout_left || is_layout_right || is_layout_stride)), reference_type> operator()(const I0& i0, const I1& i1) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; if constexpr (is_layout_left) { if constexpr (traits::rank_dynamic == 0) return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_dim.N0 * j1]; else return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_stride * j1]; } else if constexpr (is_layout_right) { if constexpr (traits::rank_dynamic == 0) return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_dim.N1 * j0]; else return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_stride * j0]; } else { static_assert(is_layout_stride); return m_map.m_impl_handle[j0 * m_map.m_impl_offset.m_stride.S0 + j1 * m_map.m_impl_offset.m_stride.S1]; } #if defined KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } //------------------------------ // Rank 3 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (3 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (3 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; return m_map.reference(j0, j1, j2); } //------------------------------ // Rank 4 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (4 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (4 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; return m_map.reference(j0, j1, j2, j3); } //------------------------------ // Rank 5 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (5 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<(Kokkos::Impl::are_integral::value && (5 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; return m_map.reference(j0, j1, j2, j3, j4); } //------------------------------ // Rank 6 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (6 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (6 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; return m_map.reference(j0, j1, j2, j3, j4, j5); } //------------------------------ // Rank 7 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (7 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5, const I6& i6) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; const size_t j6 = i6 - m_begins[6]; return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (7 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5, const I6& i6) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; const size_t j6 = i6 - m_begins[6]; return m_map.reference(j0, j1, j2, j3, j4, j5, j6); } //------------------------------ // Rank 8 template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (8 == Rank) && is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; const size_t j6 = i6 - m_begins[6]; const size_t j7 = i7 - m_begins[7]; return m_map .m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6, j7)]; } template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< (Kokkos::Impl::are_integral::value && (8 == Rank) && !is_default_map), reference_type> operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) const size_t j0 = i0 - m_begins[0]; const size_t j1 = i1 - m_begins[1]; const size_t j2 = i2 - m_begins[2]; const size_t j3 = i3 - m_begins[3]; const size_t j4 = i4 - m_begins[4]; const size_t j5 = i5 - m_begins[5]; const size_t j6 = i6 - m_begins[6]; const size_t j7 = i7 - m_begins[7]; return m_map.reference(j0, j1, j2, j3, j4, j5, j6, j7); } #undef KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY //---------------------------------------- // Standard destructor, constructors, and assignment operators KOKKOS_DEFAULTED_FUNCTION ~OffsetView() = default; KOKKOS_FUNCTION OffsetView() : m_track(), m_map() { for (size_t i = 0; i < Rank; ++i) m_begins[i] = KOKKOS_INVALID_OFFSET; } KOKKOS_FUNCTION OffsetView(const OffsetView& rhs) : m_track(rhs.m_track, traits::is_managed), m_map(rhs.m_map), m_begins(rhs.m_begins) {} KOKKOS_FUNCTION OffsetView(OffsetView&& rhs) : m_track(std::move(rhs.m_track)), m_map(std::move(rhs.m_map)), m_begins(std::move(rhs.m_begins)) {} KOKKOS_FUNCTION OffsetView& operator=(const OffsetView& rhs) { m_track = rhs.m_track; m_map = rhs.m_map; m_begins = rhs.m_begins; return *this; } KOKKOS_FUNCTION OffsetView& operator=(OffsetView&& rhs) { m_track = std::move(rhs.m_track); m_map = std::move(rhs.m_map); m_begins = std::move(rhs.m_begins); return *this; } // interoperability with View private: using view_type = View; public: KOKKOS_FUNCTION view_type view() const { view_type v(m_track, m_map); return v; } template KOKKOS_FUNCTION OffsetView(const View& aview) : m_track(aview.impl_track()), m_map() { using SrcTraits = typename OffsetView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible OffsetView copy construction"); Mapping::assign(m_map, aview.impl_map(), m_track); for (size_t i = 0; i < View::rank(); ++i) { m_begins[i] = 0; } } template KOKKOS_FUNCTION OffsetView(const View& aview, const index_list_type& minIndices) : m_track(aview.impl_track()), m_map() { using SrcTraits = typename OffsetView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible OffsetView copy construction"); Mapping::assign(m_map, aview.impl_map(), m_track); KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( traits::rank_dynamic, Rank, minIndices, label());)) KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( traits::rank_dynamic, Rank, minIndices);)) for (size_t i = 0; i < minIndices.size(); ++i) { m_begins[i] = minIndices.begin()[i]; } } template KOKKOS_FUNCTION OffsetView(const View& aview, const begins_type& beg) : m_track(aview.impl_track()), m_map(), m_begins(beg) { using SrcTraits = typename OffsetView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible OffsetView copy construction"); Mapping::assign(m_map, aview.impl_map(), m_track); } // may assign unmanaged from managed. template KOKKOS_FUNCTION OffsetView(const OffsetView& rhs) : m_track(rhs.m_track, traits::is_managed), m_map(), m_begins(rhs.m_begins) { using SrcTraits = typename OffsetView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible OffsetView copy construction"); Mapping::assign(m_map, rhs.m_map, rhs.m_track); // swb what about assign? } private: enum class subtraction_failure { none, negative, overflow, }; // Subtraction should return a non-negative number and not overflow KOKKOS_FUNCTION static subtraction_failure check_subtraction(int64_t lhs, int64_t rhs) { if (lhs < rhs) return subtraction_failure::negative; if (static_cast(-1) / static_cast(2) < static_cast(lhs) - static_cast(rhs)) return subtraction_failure::overflow; return subtraction_failure::none; } // Need a way to get at an element from both begins_type (aka Kokkos::Array // which doesn't have iterators) and index_list_type (aka // std::initializer_list which doesn't have .data() or operator[]). // Returns by value KOKKOS_FUNCTION static int64_t at(const begins_type& a, size_t pos) { return a[pos]; } KOKKOS_FUNCTION static int64_t at(index_list_type a, size_t pos) { return *(a.begin() + pos); } // Check that begins < ends for all elements // B, E can be begins_type and/or index_list_type template static subtraction_failure runtime_check_begins_ends_host(const B& begins, const E& ends) { std::string message; if (begins.size() != Rank) message += "begins.size() " "(" + std::to_string(begins.size()) + ")" " != Rank " "(" + std::to_string(Rank) + ")" "\n"; if (ends.size() != Rank) message += "ends.size() " "(" + std::to_string(begins.size()) + ")" " != Rank " "(" + std::to_string(Rank) + ")" "\n"; // If there are no errors so far, then arg_rank == Rank // Otherwise, check as much as possible size_t arg_rank = begins.size() < ends.size() ? begins.size() : ends.size(); for (size_t i = 0; i != arg_rank; ++i) { subtraction_failure sf = check_subtraction(at(ends, i), at(begins, i)); if (sf != subtraction_failure::none) { message += "(" "ends[" + std::to_string(i) + "]" " " "(" + std::to_string(at(ends, i)) + ")" " - " "begins[" + std::to_string(i) + "]" " " "(" + std::to_string(at(begins, i)) + ")" ")"; switch (sf) { case subtraction_failure::negative: message += " must be non-negative\n"; break; case subtraction_failure::overflow: message += " overflows\n"; break; default: break; } } } if (!message.empty()) { message = "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView\n" + message; Kokkos::Impl::throw_runtime_exception(message); } return subtraction_failure::none; } // Check the begins < ends for all elements template KOKKOS_FUNCTION static subtraction_failure runtime_check_begins_ends_device( const B& begins, const E& ends) { if (begins.size() != Rank) Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: begins has bad Rank"); if (ends.size() != Rank) Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: ends has bad Rank"); for (size_t i = 0; i != begins.size(); ++i) { switch (check_subtraction(at(ends, i), at(begins, i))) { case subtraction_failure::negative: Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: bad range"); break; case subtraction_failure::overflow: Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: range overflows"); break; default: break; } } return subtraction_failure::none; } template KOKKOS_FUNCTION static subtraction_failure runtime_check_begins_ends( const B& begins, const E& ends) { KOKKOS_IF_ON_HOST((return runtime_check_begins_ends_host(begins, ends);)) KOKKOS_IF_ON_DEVICE( (return runtime_check_begins_ends_device(begins, ends);)) } // Constructor around unmanaged data after checking begins < ends for all // elements // Each of B, E can be begins_type and/or index_list_type // Precondition: begins.size() == ends.size() == m_begins.size() == Rank template KOKKOS_FUNCTION OffsetView(const pointer_type& p, const B& begins_, const E& ends_, subtraction_failure) : m_track() // no tracking , m_map(Kokkos::Impl::ViewCtorProp(p), typename traits::array_layout( Rank > 0 ? at(ends_, 0) - at(begins_, 0) : 0, Rank > 1 ? at(ends_, 1) - at(begins_, 1) : 0, Rank > 2 ? at(ends_, 2) - at(begins_, 2) : 0, Rank > 3 ? at(ends_, 3) - at(begins_, 3) : 0, Rank > 4 ? at(ends_, 4) - at(begins_, 4) : 0, Rank > 5 ? at(ends_, 5) - at(begins_, 5) : 0, Rank > 6 ? at(ends_, 6) - at(begins_, 6) : 0, Rank > 7 ? at(ends_, 7) - at(begins_, 7) : 0)) { for (size_t i = 0; i != m_begins.size(); ++i) { m_begins[i] = at(begins_, i); }; } public: // Constructor around unmanaged data // Four overloads, as both begins and ends can be either // begins_type or index_list_type KOKKOS_FUNCTION OffsetView(const pointer_type& p, const begins_type& begins_, const begins_type& ends_) : OffsetView(p, begins_, ends_, runtime_check_begins_ends(begins_, ends_)) {} KOKKOS_FUNCTION OffsetView(const pointer_type& p, const begins_type& begins_, index_list_type ends_) : OffsetView(p, begins_, ends_, runtime_check_begins_ends(begins_, ends_)) {} KOKKOS_FUNCTION OffsetView(const pointer_type& p, index_list_type begins_, const begins_type& ends_) : OffsetView(p, begins_, ends_, runtime_check_begins_ends(begins_, ends_)) {} KOKKOS_FUNCTION OffsetView(const pointer_type& p, index_list_type begins_, index_list_type ends_) : OffsetView(p, begins_, ends_, runtime_check_begins_ends(begins_, ends_)) {} //---------------------------------------- // Allocation tracking properties KOKKOS_FUNCTION int use_count() const { return m_track.use_count(); } const std::string label() const { return m_track.template get_label(); } // Choosing std::pair as type for the arguments allows constructing an // OffsetView using list initialization syntax, e.g., // OffsetView dummy("dummy", {-1, 3}, {-2,2}); // We could allow arbitrary types RangeType that support // std::get<{0,1}>(RangeType const&) with std::tuple_size::value==2 // but this wouldn't allow using the syntax in the example above. template explicit OffsetView( const Label& arg_label, std::enable_if_t::value, const std::pair> range0, const std::pair range1 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range2 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range3 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range4 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range5 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range6 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range7 = KOKKOS_INVALID_INDEX_RANGE ) : OffsetView( Kokkos::Impl::ViewCtorProp(arg_label), typename traits::array_layout(range0.second - range0.first + 1, range1.second - range1.first + 1, range2.second - range2.first + 1, range3.second - range3.first + 1, range4.second - range4.first + 1, range5.second - range5.first + 1, range6.second - range6.first + 1, range7.second - range7.first + 1), {range0.first, range1.first, range2.first, range3.first, range4.first, range5.first, range6.first, range7.first}) {} template explicit OffsetView( const Kokkos::Impl::ViewCtorProp& arg_prop, const std::pair range0 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range1 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range2 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range3 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range4 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range5 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range6 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range7 = KOKKOS_INVALID_INDEX_RANGE) : OffsetView( arg_prop, typename traits::array_layout(range0.second - range0.first + 1, range1.second - range1.first + 1, range2.second - range2.first + 1, range3.second - range3.first + 1, range4.second - range4.first + 1, range5.second - range5.first + 1, range6.second - range6.first + 1, range7.second - range7.first + 1), {range0.first, range1.first, range2.first, range3.first, range4.first, range5.first, range6.first, range7.first}) {} template explicit KOKKOS_FUNCTION OffsetView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout, const index_list_type minIndices) : m_track() // No memory tracking , m_map(arg_prop, arg_layout) { for (size_t i = 0; i < minIndices.size(); ++i) { m_begins[i] = minIndices.begin()[i]; } static_assert( std::is_same::pointer_type>::value, "When constructing OffsetView to wrap user memory, you must supply " "matching pointer type"); } template explicit OffsetView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout, const index_list_type minIndices) : m_track(), m_map() { for (size_t i = 0; i < Rank; ++i) m_begins[i] = minIndices.begin()[i]; // Copy the input allocation properties with possibly defaulted properties auto prop_copy = Kokkos::Impl::with_properties_if_unset( arg_prop, std::string{}, typename traits::device_type::memory_space{}, typename traits::device_type::execution_space{}); using alloc_prop = decltype(prop_copy); static_assert(traits::is_managed, "OffsetView allocation constructor requires managed memory"); if (alloc_prop::initialize && !alloc_prop::execution_space::impl_is_initialized()) { // If initializing view data then // the execution space must be initialized. Kokkos::Impl::throw_runtime_exception( "Constructing OffsetView and initializing data with uninitialized " "execution space"); } Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( prop_copy, arg_layout, Kokkos::Impl::ViewCtorProp::has_execution_space); // Setup and initialization complete, start tracking m_track.assign_allocated_record_to_uninitialized(record); KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( traits::rank_dynamic, Rank, minIndices, label());)) KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( traits::rank_dynamic, Rank, minIndices);)) } }; /** \brief Temporary free function rank() * until rank() is implemented * in the View */ template KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const OffsetView& V) { return V.Rank; } // Temporary until added to view //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Impl { template KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> shift_input(const T arg, const int64_t offset) { return arg - offset; } KOKKOS_INLINE_FUNCTION Kokkos::ALL_t shift_input(const Kokkos::ALL_t arg, const int64_t /*offset*/) { return arg; } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value, Kokkos::pair> shift_input(const Kokkos::pair arg, const int64_t offset) { return Kokkos::make_pair(arg.first - offset, arg.second - offset); } template inline std::enable_if_t::value, std::pair> shift_input(const std::pair arg, const int64_t offset) { return std::make_pair(arg.first - offset, arg.second - offset); } template KOKKOS_INLINE_FUNCTION void map_arg_to_new_begin( const size_t i, Kokkos::Array& subviewBegins, std::enable_if_t shiftedArg, const Arg arg, const A viewBegins, size_t& counter) { if (!std::is_integral::value) { subviewBegins[counter] = shiftedArg == arg ? viewBegins[i] : 0; counter++; } } template KOKKOS_INLINE_FUNCTION void map_arg_to_new_begin( const size_t /*i*/, Kokkos::Array& /*subviewBegins*/, std::enable_if_t /*shiftedArg*/, const Arg /*arg*/, const A /*viewBegins*/, size_t& /*counter*/) {} template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping, T>::type>::type subview_offset(const OffsetView& src, T arg) { auto theView = src.view(); auto begins = src.begins(); T shiftedArg = shift_input(arg, begins[0]); constexpr size_t rank = Kokkos::Impl::ViewMapping, T>::type::rank; auto theSubview = Kokkos::subview(theView, shiftedArg); Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin(0, subviewBegins, shiftedArg, arg, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping, T>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1>::type>::type subview_offset(const Kokkos::Experimental::OffsetView& src, T0 arg0, T1 arg1) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1); constexpr size_t rank = Kokkos::Impl::ViewMapping, T0, T1>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2); constexpr size_t rank = Kokkos::Impl::ViewMapping, T0, T1, T2>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, T3 arg3) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); T3 shiftedArg3 = shift_input(arg3, begins[3]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, shiftedArg3); constexpr size_t rank = Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 3, subviewBegins, shiftedArg3, arg3, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, T3 arg3, T4 arg4) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); T3 shiftedArg3 = shift_input(arg3, begins[3]); T4 shiftedArg4 = shift_input(arg4, begins[4]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, shiftedArg3, shiftedArg4); constexpr size_t rank = Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 3, subviewBegins, shiftedArg3, arg3, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 4, subviewBegins, shiftedArg4, arg4, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); T3 shiftedArg3 = shift_input(arg3, begins[3]); T4 shiftedArg4 = shift_input(arg4, begins[4]); T5 shiftedArg5 = shift_input(arg5, begins[5]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, shiftedArg3, shiftedArg4, shiftedArg5); constexpr size_t rank = Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 3, subviewBegins, shiftedArg3, arg3, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 4, subviewBegins, shiftedArg4, arg4, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 5, subviewBegins, shiftedArg5, arg5, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); T3 shiftedArg3 = shift_input(arg3, begins[3]); T4 shiftedArg4 = shift_input(arg4, begins[4]); T5 shiftedArg5 = shift_input(arg5, begins[5]); T6 shiftedArg6 = shift_input(arg6, begins[6]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, shiftedArg3, shiftedArg4, shiftedArg5, shiftedArg6); constexpr size_t rank = Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 3, subviewBegins, shiftedArg3, arg3, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 4, subviewBegins, shiftedArg4, arg4, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 5, subviewBegins, shiftedArg5, arg5, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 6, subviewBegins, shiftedArg6, arg6, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type>::type subview_offset(const OffsetView& src, T0 arg0, T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6, T7 arg7) { auto theView = src.view(); auto begins = src.begins(); T0 shiftedArg0 = shift_input(arg0, begins[0]); T1 shiftedArg1 = shift_input(arg1, begins[1]); T2 shiftedArg2 = shift_input(arg2, begins[2]); T3 shiftedArg3 = shift_input(arg3, begins[3]); T4 shiftedArg4 = shift_input(arg4, begins[4]); T5 shiftedArg5 = shift_input(arg5, begins[5]); T6 shiftedArg6 = shift_input(arg6, begins[6]); T7 shiftedArg7 = shift_input(arg7, begins[7]); auto theSubview = Kokkos::subview(theView, shiftedArg0, shiftedArg1, shiftedArg2, shiftedArg3, shiftedArg4, shiftedArg5, shiftedArg6, shiftedArg7); constexpr size_t rank = Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type::rank; Kokkos::Array subviewBegins; size_t counter = 0; Kokkos::Experimental::Impl::map_arg_to_new_begin( 0, subviewBegins, shiftedArg0, arg0, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 1, subviewBegins, shiftedArg1, arg1, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 2, subviewBegins, shiftedArg2, arg2, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 3, subviewBegins, shiftedArg3, arg3, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 4, subviewBegins, shiftedArg4, arg4, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 5, subviewBegins, shiftedArg5, arg5, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 6, subviewBegins, shiftedArg6, arg6, begins, counter); Kokkos::Experimental::Impl::map_arg_to_new_begin( 7, subviewBegins, shiftedArg7, arg7, begins, counter); typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, T0, T1, T2, T3, T4, T5, T6, T7>::type>::type offsetView(theSubview, subviewBegins); return offsetView; } } // namespace Impl template KOKKOS_INLINE_FUNCTION typename Kokkos::Experimental::Impl::GetOffsetViewTypeFromViewType< typename Kokkos::Impl::ViewMapping< void /* deduce subview type from source view traits */ , ViewTraits, Args...>::type>::type subview(const OffsetView& src, Args... args) { static_assert( OffsetView::Rank == sizeof...(Args), "subview requires one argument for each source OffsetView rank"); return Kokkos::Experimental::Impl::subview_offset(src, args...); } } // namespace Experimental } // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { namespace Experimental { template KOKKOS_INLINE_FUNCTION bool operator==(const OffsetView& lhs, const OffsetView& rhs) { // Same data, layout, dimensions using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; return std::is_same::value && std::is_same::value && std::is_same::value && unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7) && lhs.begin(0) == rhs.begin(0) && lhs.begin(1) == rhs.begin(1) && lhs.begin(2) == rhs.begin(2) && lhs.begin(3) == rhs.begin(3) && lhs.begin(4) == rhs.begin(4) && lhs.begin(5) == rhs.begin(5) && lhs.begin(6) == rhs.begin(6) && lhs.begin(7) == rhs.begin(7); } template KOKKOS_INLINE_FUNCTION bool operator!=(const OffsetView& lhs, const OffsetView& rhs) { return !(operator==(lhs, rhs)); } template KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, const OffsetView& rhs) { // Same data, layout, dimensions using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; return std::is_same::value && std::is_same::value && std::is_same::value && unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); } template KOKKOS_INLINE_FUNCTION bool operator==(const OffsetView& lhs, const View& rhs) { return rhs == lhs; } } // namespace Experimental } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { template inline void deep_copy( const Experimental::OffsetView& dst, typename ViewTraits::const_value_type& value, std::enable_if_t::specialize, void>::value>* = nullptr) { static_assert( std::is_same::non_const_value_type, typename ViewTraits::value_type>::value, "deep_copy requires non-const type"); auto dstView = dst.view(); Kokkos::deep_copy(dstView, value); } template inline void deep_copy( const Experimental::OffsetView& dst, const Experimental::OffsetView& value, std::enable_if_t::specialize, void>::value>* = nullptr) { static_assert( std::is_same::value_type, typename ViewTraits::non_const_value_type>::value, "deep_copy requires matching non-const destination type"); auto dstView = dst.view(); Kokkos::deep_copy(dstView, value.view()); } template inline void deep_copy( const Experimental::OffsetView& dst, const View& value, std::enable_if_t::specialize, void>::value>* = nullptr) { static_assert( std::is_same::value_type, typename ViewTraits::non_const_value_type>::value, "deep_copy requires matching non-const destination type"); auto dstView = dst.view(); Kokkos::deep_copy(dstView, value); } template inline void deep_copy( const View& dst, const Experimental::OffsetView& value, std::enable_if_t::specialize, void>::value>* = nullptr) { static_assert( std::is_same::value_type, typename ViewTraits::non_const_value_type>::value, "deep_copy requires matching non-const destination type"); Kokkos::deep_copy(dst, value.view()); } namespace Impl { // Deduce Mirror Types template struct MirrorOffsetViewType { // The incoming view_type using src_view_type = typename Kokkos::Experimental::OffsetView; // The memory space for the mirror view using memory_space = typename Space::memory_space; // Check whether it is the same memory space enum { is_same_memspace = std::is_same::value }; // The array_layout using array_layout = typename src_view_type::array_layout; // The data type (we probably want it non-const since otherwise we can't even // deep_copy to it.) using data_type = typename src_view_type::non_const_data_type; // The destination view type if it is not the same memory space using dest_view_type = Kokkos::Experimental::OffsetView; // If it is the same memory_space return the existing view_type // This will also keep the unmanaged trait if necessary using view_type = std::conditional_t; }; template struct MirrorOffsetType { // The incoming view_type using src_view_type = typename Kokkos::Experimental::OffsetView; // The memory space for the mirror view using memory_space = typename Space::memory_space; // Check whether it is the same memory space enum { is_same_memspace = std::is_same::value }; // The array_layout using array_layout = typename src_view_type::array_layout; // The data type (we probably want it non-const since otherwise we can't even // deep_copy to it.) using data_type = typename src_view_type::non_const_data_type; // The destination view type if it is not the same memory space using view_type = Kokkos::Experimental::OffsetView; }; } // namespace Impl namespace Impl { // create a mirror // private interface that accepts arbitrary view constructor args passed by a // view_alloc template inline auto create_mirror(const Kokkos::Experimental::OffsetView& src, const Impl::ViewCtorProp& arg_prop) { check_view_ctor_args_create_mirror(); if constexpr (Impl::ViewCtorProp::has_memory_space) { using Space = typename Impl::ViewCtorProp::memory_space; auto prop_copy = Impl::with_properties_if_unset( arg_prop, std::string(src.label()).append("_mirror")); return typename Kokkos::Impl::MirrorOffsetType::view_type( prop_copy, src.layout(), {src.begin(0), src.begin(1), src.begin(2), src.begin(3), src.begin(4), src.begin(5), src.begin(6), src.begin(7)}); } else { return typename Kokkos::Experimental::OffsetView::HostMirror( Kokkos::create_mirror(arg_prop, src.view()), src.begins()); } #if defined KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } } // namespace Impl // Create a mirror in host space template inline auto create_mirror( const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror(src, Impl::ViewCtorProp<>{}); } template inline auto create_mirror( Kokkos::Impl::WithoutInitializing_t wi, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror(src, Kokkos::view_alloc(wi)); } // Create a mirror in a new space template ::value>> inline auto create_mirror( const Space&, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(typename Space::memory_space{})); } template typename Kokkos::Impl::MirrorOffsetType::view_type create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space&, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(typename Space::memory_space{}, wi)); } template inline auto create_mirror( const Impl::ViewCtorProp& arg_prop, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror(src, arg_prop); } namespace Impl { // create a mirror view // private interface that accepts arbitrary view constructor args passed by a // view_alloc template inline auto create_mirror_view( const Kokkos::Experimental::OffsetView& src, [[maybe_unused]] const Impl::ViewCtorProp& arg_prop) { if constexpr (!Impl::ViewCtorProp::has_memory_space) { if constexpr (std::is_same::memory_space, typename Kokkos::Experimental::OffsetView< T, P...>::HostMirror::memory_space>::value && std::is_same::data_type, typename Kokkos::Experimental::OffsetView< T, P...>::HostMirror::data_type>::value) { return typename Kokkos::Experimental::OffsetView::HostMirror(src); } else { return Kokkos::Impl::create_mirror(src, arg_prop); } } else { if constexpr (Impl::MirrorOffsetViewType::memory_space, T, P...>::is_same_memspace) { return typename Impl::MirrorOffsetViewType< typename Impl::ViewCtorProp::memory_space, T, P...>::view_type(src); } else { return Kokkos::Impl::create_mirror(src, arg_prop); } } #if defined KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } } // namespace Impl // Create a mirror view in host space template inline auto create_mirror_view( const typename Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror_view(src, Impl::ViewCtorProp<>{}); } template inline auto create_mirror_view( Kokkos::Impl::WithoutInitializing_t wi, const typename Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror_view(src, Kokkos::view_alloc(wi)); } // Create a mirror view in a new space template ::value>> inline auto create_mirror_view( const Space&, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror_view( src, Kokkos::view_alloc(typename Space::memory_space{})); } template inline auto create_mirror_view( Kokkos::Impl::WithoutInitializing_t wi, const Space&, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror_view( src, Kokkos::view_alloc(typename Space::memory_space{}, wi)); } template inline auto create_mirror_view( const Impl::ViewCtorProp& arg_prop, const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror_view(src, arg_prop); } // Create a mirror view and deep_copy in a new space template typename Kokkos::Impl::MirrorOffsetViewType< typename Impl::ViewCtorProp::memory_space, T, P...>::view_type create_mirror_view_and_copy( const Impl::ViewCtorProp& arg_prop, const Kokkos::Experimental::OffsetView& src) { return {create_mirror_view_and_copy(arg_prop, src.view()), src.begins()}; } template typename Kokkos::Impl::MirrorOffsetViewType::view_type create_mirror_view_and_copy( const Space& space, const Kokkos::Experimental::OffsetView& src, std::string const& name = "") { return {create_mirror_view_and_copy(space, src.view(), name), src.begins()}; } } /* namespace Kokkos */ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- #ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW #undef KOKKOS_IMPL_PUBLIC_INCLUDE #undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_OFFSETVIEW #endif #endif /* KOKKOS_OFFSETVIEW_HPP_ */