gridtools-1.1.3/0000755000175000017500000000000013613316064013707 5ustar alastairalastairgridtools-1.1.3/.clang-format0000644000175000017500000000342013613316060016255 0ustar alastairalastair--- Language: Cpp # BasedOnStyle: LLVM AccessModifierOffset: -2 ConstructorInitializerIndentWidth: 4 #AlignEscapedNewlinesLeft: false AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AlwaysBreakTemplateDeclarations: true AlwaysBreakBeforeMultilineStrings: false BreakBeforeBinaryOperators: false BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BinPackParameters: false BinPackArguments: false ColumnLimit: 120 ConstructorInitializerAllOnOneLineOrOnePerLine: false DerivePointerAlignment: false ExperimentalAutoDetectBinPacking: false IndentCaseLabels: false IndentWrappedFunctionNames: false IndentFunctionDeclarationAfterType: false MaxEmptyLinesToKeep: 1 KeepEmptyLinesAtTheStartOfBlocks: true NamespaceIndentation: All ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: true PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 300 PenaltyBreakString: 1000 PenaltyBreakFirstLessLess: 120 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right SpacesBeforeTrailingComments: 1 Cpp11BracedListStyle: true Standard: Cpp11 IndentWidth: 4 TabWidth: 8 UseTab: Never BreakBeforeBraces: Attach SpacesInParentheses: false SpacesInAngles: false SpaceInEmptyParentheses: false SpacesInCStyleCastParentheses: false SpacesInContainerLiterals: true SpaceBeforeAssignmentOperators: true ContinuationIndentWidth: 4 CommentPragmas: '*' ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] SpaceBeforeParens: ControlStatements DisableFormat: false AlignAfterOpenBracket: false AlignEscapedNewlinesLeft: true ... gridtools-1.1.3/src/0000755000175000017500000000000013613316060014472 5ustar alastairalastairgridtools-1.1.3/src/CMakeLists.txt0000644000175000017500000000051513613316060017233 0ustar alastairalastairadd_library(gcl GCL.cpp ) target_link_libraries(gcl PUBLIC gridtools) if(GT_USE_MPI) target_link_libraries(gcl PUBLIC MPI::MPI_CXX) endif() install(TARGETS gcl EXPORT GridToolsTargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin INCLUDES DESTINATION include ) add_subdirectory(tools) gridtools-1.1.3/src/tools/0000755000175000017500000000000013613316060015632 5ustar alastairalastairgridtools-1.1.3/src/tools/CMakeLists.txt0000644000175000017500000000021513613316060020370 0ustar alastairalastairif(BUILD_TESTING) add_library(regression_main regression_fixture.cpp) target_link_libraries(regression_main gtest gridtools) endif() gridtools-1.1.3/src/tools/regression_fixture.cpp0000644000175000017500000000402313613316060022263 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #include #include #include #include #include #include #include namespace gridtools { namespace _impl { uint_t regression_fixture_base::s_d1 = 0; uint_t regression_fixture_base::s_d2 = 0; uint_t regression_fixture_base::s_d3 = 0; uint_t regression_fixture_base::s_steps = 0; bool regression_fixture_base::s_needs_verification = true; void regression_fixture_base::flush_cache() { static std::size_t n = 1024 * 1024 * 21 / 2; static std::vector a_(n), b_(n), c_(n); double *a = a_.data(); double *b = b_.data(); double *c = c_.data(); #pragma omp parallel for for (std::size_t i = 0; i < n; i++) a[i] = b[i] * c[i]; } void regression_fixture_base::init(int argc, char **argv) { if (argc < 4) { std::cerr << "Usage: " << argv[0] << " " << "dimx dimy dimz tsteps\n\twhere args are integer sizes of the data fields and tsteps " "is the number of time steps to run in a benchmark run" << std::endl; exit(1); } s_d1 = std::atoi(argv[1]); s_d2 = std::atoi(argv[2]); s_d3 = std::atoi(argv[3]); s_steps = argc > 4 ? std::atoi(argv[4]) : 0; s_needs_verification = argc < 6 || std::strcmp(argv[5], "-d") != 0; } } // namespace _impl } // namespace gridtools int main(int argc, char **argv) { // Pass command line arguments to googltest ::testing::InitGoogleTest(&argc, argv); gridtools::_impl::regression_fixture_base::init(argc, argv); return RUN_ALL_TESTS(); } gridtools-1.1.3/src/GCL.cpp0000644000175000017500000000622713613316060015612 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #include #ifdef GCL_GPU #ifdef GCL_MULTI_STREAMS #ifdef GCL_USE_3 cudaStream_t ZL_stream; cudaStream_t &ZU_stream = ZL_stream; cudaStream_t YL_stream; cudaStream_t &YU_stream = YL_stream; cudaStream_t XL_stream; cudaStream_t &XU_stream = XL_stream; #else cudaStream_t ZL_stream; cudaStream_t ZU_stream; cudaStream_t YL_stream; cudaStream_t YU_stream; cudaStream_t XL_stream; cudaStream_t XU_stream; #endif #endif #endif namespace gridtools { #ifdef GCL_MPI MPI_Comm GCL_WORLD; int PID; int PROCS; namespace _impl { void GCL_Real_Init(int argc, char **argv) { int ready; MPI_Initialized(&ready); if (!ready) MPI_Init(&argc, &argv); GCL_WORLD = MPI_COMM_WORLD; MPI_Comm_rank(GCL_WORLD, &PID); MPI_Comm_size(GCL_WORLD, &PROCS); #ifdef GCL_MULTI_STREAMS #ifdef GCL_USE_3 GT_CUDA_CHECK(cudaStreamCreate(&ZL_stream)); GT_CUDA_CHECK(cudaStreamCreate(&YL_stream)); GT_CUDA_CHECK(cudaStreamCreate(&XL_stream)); #else GT_CUDA_CHECK(cudaStreamCreate(&ZL_stream)); GT_CUDA_CHECK(cudaStreamCreate(&ZU_stream)); GT_CUDA_CHECK(cudaStreamCreate(&YL_stream)); GT_CUDA_CHECK(cudaStreamCreate(&YU_stream)); GT_CUDA_CHECK(cudaStreamCreate(&XL_stream)); GT_CUDA_CHECK(cudaStreamCreate(&XU_stream)); #endif #endif } } // namespace _impl void GCL_Init(int argc, char **argv) { _impl::GCL_Real_Init(argc, argv); } void GCL_Init() { int arg = 1; _impl::GCL_Real_Init(arg, 0); } void GCL_Finalize() { #ifdef GCL_MULTI_STREAMS #ifdef GCL_USE_3 GT_CUDA_CHECK(cudaStreamDestroy(ZL_stream)); GT_CUDA_CHECK(cudaStreamDestroy(YL_stream)); GT_CUDA_CHECK(cudaStreamDestroy(XL_stream)); #else GT_CUDA_CHECK(cudaStreamDestroy(ZL_stream)); GT_CUDA_CHECK(cudaStreamDestroy(ZU_stream)); GT_CUDA_CHECK(cudaStreamDestroy(YL_stream)); GT_CUDA_CHECK(cudaStreamDestroy(YU_stream)); GT_CUDA_CHECK(cudaStreamDestroy(XL_stream)); GT_CUDA_CHECK(cudaStreamDestroy(XU_stream)); #endif #endif MPI_Finalize(); } #ifdef GCL_TRACE // initialize static instance_ to nullptr template <> stats_collector<1> *stats_collector<1>::instance_ = 0; template <> stats_collector<2> *stats_collector<2>::instance_ = 0; template <> stats_collector<3> *stats_collector<3>::instance_ = 0; // convenient handles for the singleton instances for 2D and 3D grids stats_collector<3> &stats_collector_3D = *stats_collector<3>::instance(); stats_collector<2> &stats_collector_2D = *stats_collector<2>::instance(); #endif #else int GCL_WORLD; int PID; int PROCS; void GCL_Init(int, char **) { PROCS = 1; PID = 0; } void GCL_Init() { PROCS = 1; PID = 0; } void GCL_Finalize() {} #endif } // namespace gridtools gridtools-1.1.3/include/0000755000175000017500000000000013613316060015326 5ustar alastairalastairgridtools-1.1.3/include/gridtools/0000755000175000017500000000000013613316060017334 5ustar alastairalastairgridtools-1.1.3/include/gridtools/storage/0000755000175000017500000000000013613316060021000 5ustar alastairalastairgridtools-1.1.3/include/gridtools/storage/data_view.hpp0000644000175000017500000002242513613316060023461 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include #include "../common/array.hpp" #include "../common/defs.hpp" #include "../common/gt_assert.hpp" #include "../common/host_device.hpp" #include "../meta/type_traits.hpp" #include "common/definitions.hpp" #include "data_store.hpp" #ifndef GT_CHECK_MEMORY_SPACE #ifdef __CUDA_ARCH__ #define GT_CHECK_MEMORY_SPACE(device_view) \ GT_ASSERT_OR_THROW(device_view, "can not access a host view from within a GPU kernel") #else #define GT_CHECK_MEMORY_SPACE(device_view) \ GT_ASSERT_OR_THROW(!device_view, "can not access a device view from a host function") #endif #endif namespace gridtools { /** \ingroup storage * @{ */ /** * @brief data_view implementation. This struct provides means to modify contents of * gridtools data_store containers on arbitrary locations (host, device, etc.). * @tparam DataStore data store type * @tparam AccessMode access mode (default is read-write) */ template struct data_view { GT_STATIC_ASSERT(is_data_store::value, GT_INTERNAL_ERROR_MSG("Passed type is no data_store type")); using data_store_t = DataStore; typedef typename DataStore::data_t data_t; typedef typename DataStore::state_machine_t state_machine_t; typedef typename DataStore::storage_info_t storage_info_t; const static access_mode mode = AccessMode; private: data_t *m_raw_ptr; state_machine_t *m_state_machine_ptr; storage_info_t const *m_storage_info; bool m_device_view; public: /** * @brief data_view constructor */ GT_FUNCTION data_view() : m_raw_ptr(nullptr), m_state_machine_ptr(nullptr), m_storage_info(nullptr), m_device_view(false) {} /** * @brief data_view constructor. This constructor is normally not called by the user because it is more * convenient to use the provided make functions. * @param data_ptr pointer to the data * @param info_ptr pointer to the storage_info * @param state_ptr pointer to the state machine * @param device_view true if device view, false otherwise */ GT_FUNCTION data_view( data_t *data_ptr, storage_info_t const *info_ptr, state_machine_t *state_ptr, bool device_view) : m_raw_ptr(data_ptr), m_state_machine_ptr(state_ptr), m_storage_info(info_ptr), m_device_view(device_view) { GT_ASSERT_OR_THROW(data_ptr, "Cannot create data_view with invalid data pointer"); GT_ASSERT_OR_THROW(info_ptr, "Cannot create data_view with invalid storage info pointer"); } GT_FUNCTION storage_info_t const &storage_info() const { GT_CHECK_MEMORY_SPACE(m_device_view); return *m_storage_info; } /** * data getter */ GT_FUNCTION data_t *data() { return m_raw_ptr; } /** * data getter */ GT_FUNCTION data_t const *data() const { return m_raw_ptr; } /** * @return pointer to the first position */ GT_FUNCTION data_t *ptr_to_first_position() { return &operator()(gridtools::array{{}}); } /** * return pointer to the first position */ GT_FUNCTION data_t const *ptr_to_first_position() const { return &operator()(gridtools::array{{}}); } /** * @brief operator() is used to access elements. E.g., view(0,0,2) will return the third element. * @param c given indices * @return reference to the queried value */ template std::conditional_t GT_FUNCTION operator()( Coords... c) const { GT_STATIC_ASSERT(conjunction>::value, GT_INTERNAL_ERROR_MSG("Index arguments have to be integral types.")); GT_CHECK_MEMORY_SPACE(m_device_view); return m_raw_ptr[m_storage_info->index(c...)]; } /** * @brief operator() is used to access elements. E.g., view({0,0,2}) will return the third element. * @param arr array of indices * @return reference to the queried value */ std::conditional_t GT_FUNCTION operator()( gridtools::array const &arr) const { GT_CHECK_MEMORY_SPACE(m_device_view); return m_raw_ptr[m_storage_info->index(arr)]; } /** * @brief Check if view contains valid pointers, and simple state machine checks. * Be aware that this is not a full check. In order to check if a view is in a * consistent state use check_consistency function. * @return true if pointers and state is correct, otherwise false */ bool valid() const { // ptrs invalid -> view invalid if (!m_raw_ptr || !m_storage_info) return false; // when used in combination with a host storage the view is always valid as long as the ptrs are if (!m_state_machine_ptr) return true; // read only -> simple check if (AccessMode == access_mode::read_only) return m_device_view ? !m_state_machine_ptr->m_dnu : !m_state_machine_ptr->m_hnu; else // check state machine ptrs return m_device_view ? ((m_state_machine_ptr->m_hnu) && !(m_state_machine_ptr->m_dnu)) : (!(m_state_machine_ptr->m_hnu) && (m_state_machine_ptr->m_dnu)); } /* * @brief member function to retrieve the total size (dimensions, halos, padding, initial_offset). * @return total size */ GT_FUNCTION GT_CONSTEXPR int padded_total_length() const { return m_storage_info->padded_total_length(); } /* * @brief Returns the length of a dimension excluding the halo points (only the inner region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int length() const { return m_storage_info->template length(); } /* * @brief Returns the length of a dimension including the halo points (the outer region) * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int total_length() const { return m_storage_info->template total_length(); } /* * @brief Returns the index of the first element in the specified dimension when iterating in the whole outer * region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int total_begin() const { return m_storage_info->template total_begin(); } /* * @brief Returns the index of the first element in the specified dimension when iterating in the inner region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int begin() const { return m_storage_info->template begin(); } /* * @brief Returns the index of the last element in the specified dimension when iterating in the whole outer * region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int total_end() const { return m_storage_info->template total_end(); } /* * @brief Returns the index of the last element in the specified dimension when iterating in the inner region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int end() const { return m_storage_info->template end(); } friend data_t *advanced_get_raw_pointer_of(data_view const &src) { return src.m_raw_ptr; } }; template struct is_data_view : std::false_type {}; template struct is_data_view> : std::true_type {}; namespace advanced { /** Function to access the protected data member of the views containing the raw pointers. This is an interface we want to avoid using (and future fixed of the communication library will not need this. We made the use of this function difficult on purpose. */ template auto get_raw_pointer_of(T const &src) { return advanced_get_raw_pointer_of(src); } } // namespace advanced /** * @} */ } // namespace gridtools #undef GT_CHECK_MEMORY_SPACE gridtools-1.1.3/include/gridtools/storage/storage_traits_mc.hpp0000644000175000017500000000731113613316060025224 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include "../common/defs.hpp" #include "../common/layout_map.hpp" #include "../common/selector.hpp" #include "./common/halo.hpp" #include "./common/storage_traits_metafunctions.hpp" #include "./storage_mc/mc_storage.hpp" #include "./storage_mc/mc_storage_info.hpp" namespace gridtools { template struct storage_traits_from_id; namespace impl { template struct layout_swap_mc { using type = LayoutMap; }; template struct layout_swap_mc> { using type = layout_map; }; } // namespace impl /** @brief storage traits for the Mic backend*/ template <> struct storage_traits_from_id { template struct select_storage { using type = mc_storage; }; template struct select_storage_info { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); using layout = typename impl::layout_swap_mc::type>::type; using type = mc_storage_info; }; template struct select_custom_layout_storage_info { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); GT_STATIC_ASSERT(is_layout_map::value, "Given type is not a layout map type."); using type = mc_storage_info; }; template struct select_special_storage_info { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); GT_STATIC_ASSERT(is_selector::value, "Given type is not a selector type."); using layout = typename impl::layout_swap_mc::type>::type; using type = mc_storage_info::type, Halo>; }; template struct select_storage_info_align { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); using layout = typename impl::layout_swap_mc::type>::type; using type = storage_info; }; template struct select_custom_layout_storage_info_align { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); GT_STATIC_ASSERT(is_layout_map::value, "Given type is not a layout map type."); using type = storage_info; }; template struct select_special_storage_info_align { GT_STATIC_ASSERT(is_halo::value, "Given type is not a halo type."); GT_STATIC_ASSERT(is_selector::value, "Given type is not a selector type."); using layout = typename impl::layout_swap_mc::type>::type; using type = storage_info::type, Halo, Align>; }; }; } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/storage_facility.hpp0000644000175000017500000000577213613316060025054 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include "../common/layout_map.hpp" #include "common/definitions.hpp" #include "common/halo.hpp" #include "data_store.hpp" #ifdef GT_USE_GPU #include "storage_traits_cuda.hpp" #include "storage_cuda/data_view_helpers.hpp" #endif #include "storage_traits_mc.hpp" #include "storage_traits_naive.hpp" #include "storage_traits_x86.hpp" #include "storage_host/data_view_helpers.hpp" #include "storage_mc/data_view_helpers.hpp" /** * \defgroup storage Storage */ namespace gridtools { /** \ingroup storage * @{ */ /** * @brief storage traits used to retrieve the correct storage_info, data_store, and data_store_field types. * Additionally to the default types, specialized and custom storage_info types can be retrieved * @tparam T used target (e.g., Cuda or Host) */ template struct storage_traits : gridtools::storage_traits_from_id { private: template using storage_t = typename gridtools::storage_traits_from_id::template select_storage::type; public: template > using storage_info_t = typename gridtools::storage_traits_from_id::template select_storage_info::type; template > using custom_layout_storage_info_t = typename gridtools::storage_traits_from_id< Backend>::template select_custom_layout_storage_info::type; template > using special_storage_info_t = typename gridtools::storage_traits_from_id< Backend>::template select_special_storage_info::type; template using data_store_t = data_store, StorageInfo>; template using storage_info_align_t = typename gridtools::storage_traits_from_id< Backend>::template select_storage_info_align::type; template using custom_layout_storage_info_align_t = typename gridtools::storage_traits_from_id< Backend>::template select_custom_layout_storage_info_align::type; template using special_storage_info_align_t = typename gridtools::storage_traits_from_id< Backend>::template select_special_storage_info_align::type; }; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/0000755000175000017500000000000013613316060022270 5ustar alastairalastairgridtools-1.1.3/include/gridtools/storage/common/state_machine.hpp0000644000175000017500000000146613613316060025614 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once namespace gridtools { /** \ingroup storage * @{ */ /** * @brief A class that represents the state machine that is used to determine * if a storage is currently on the host or on the device and if the * data on the host or the device is outdated and needs to be updated. */ struct state_machine { bool m_hnu; // hnu = host needs update, set to true if a non-read-only device view is instantiated. bool m_dnu; // dnu = device needs update, set to true if a non-read-only host view is instantiated. }; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/definitions.hpp0000644000175000017500000000100413613316060025307 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include "../../common/defs.hpp" #include "../../common/host_device.hpp" namespace gridtools { /** \ingroup storage * @{ */ enum class ownership { external_gpu, external_cpu }; enum class access_mode { read_write = 0, read_only = 1 }; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/alignment.hpp0000644000175000017500000000160113613316060024755 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include #include "../../common/defs.hpp" namespace gridtools { /** \ingroup storage * @{ */ /** * @brief This struct is used to pass alignment information to storage info types. Alignment is in terms of number * of elements. * @tparam alignment value */ template struct alignment { GT_STATIC_ASSERT(N > 0, "Alignment value must be greater than 0"); static constexpr uint_t value = N; }; template struct is_alignment : std::false_type {}; template struct is_alignment> : std::true_type {}; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/storage_traits_metafunctions.hpp0000644000175000017500000001175513613316060031003 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include "../../common/generic_metafunctions/accumulate.hpp" #include "../../common/gt_assert.hpp" #include "../../common/layout_map.hpp" #include "../../common/selector.hpp" #include "../../meta/logical.hpp" namespace gridtools { /** \ingroup storage * @{ */ namespace _impl { /* Layout map extender, takes a given layout and extends it by n dimensions (ascending and descending version) */ template struct layout_map_ext_asc; template struct layout_map_ext_asc> : layout_map_ext_asc> {}; template struct layout_map_ext_asc<0, Current, layout_map> { typedef layout_map type; }; template struct layout_map_ext_dsc; template struct layout_map_ext_dsc> : layout_map_ext_dsc> {}; template struct layout_map_ext_dsc<0, layout_map> { typedef layout_map type; }; } // namespace _impl /* get a standard layout_map (n-dimensional and ascending or descending) */ template struct get_layout; // get a multidimensional layout in ascending order (e.g., host backend) /** * @brief metafunction used to retrieve a layout_map with n-dimensions * that can be used in combination with the host backend (k-first order). * E.g., get_layout< 5, true > will return following type: layout_map< 2, 3, 4, 0, 1 >. * This means the k-dimension (value: 4) is coalesced in memory, followed * by the j-dimension (value: 3), followed by the i-dimension (value: 2), followed * by the fifth dimension (value: 1), etc. The reason for having k as innermost * is because of the gridtools execution model. The CPU backend will give best * performance (in most cases) when using the provided layout. */ template struct get_layout { GT_STATIC_ASSERT(Dim > 0, GT_INTERNAL_ERROR_MSG("Zero dimensional layout makes no sense.")); typedef typename _impl::layout_map_ext_asc>::type type; }; // get a multidimensional layout in descending order (e.g., gpu backend) /** * @brief metafunction used to retrieve a layout_map with n-dimensions * that can be used in combination with the GPU backend (i-first order). * E.g., get_layout< 5, false > will return following type: layout_map< 4, 3, 2, 1, 0 >. * This means the i-dimension (value: 4) is coalesced in memory, followed * by the j-dimension (value: 3), followed by the k-dimension (value: 2), followed * by the fourth dimension (value: 1), etc. The reason for having i as innermost * is because of the gridtools execution model. The GPU backend will give best * performance (in most cases) when using the provided layout. */ template struct get_layout { GT_STATIC_ASSERT(Dim > 0, GT_INTERNAL_ERROR_MSG("Zero dimensional layout makes no sense.")); typedef typename _impl::layout_map_ext_dsc>::type type; }; /* specializations up to 3-dimensional for both i-first and k-first layouts */ template <> struct get_layout<1, true> { typedef layout_map<0> type; }; template <> struct get_layout<1, false> { typedef layout_map<0> type; }; template <> struct get_layout<2, true> { typedef layout_map<0, 1> type; }; template <> struct get_layout<2, false> { typedef layout_map<1, 0> type; }; template <> struct get_layout<3, true> { typedef layout_map<0, 1, 2> type; }; template <> struct get_layout<3, false> { typedef layout_map<2, 1, 0> type; }; /** * @brief metafunction used to retrieve special layout_map. * Special layout_map are layout_maps with masked dimensions. * @tparam T the layout_map type * @tparam Selector the selector type */ template struct get_special_layout; template struct get_special_layout, selector> { static constexpr int correction(int D) { return accumulate(plus_functor{}, (!Bitmask && Dims >= 0 && Dims < D ? 1 : 0)...); } using type = layout_map<(Bitmask ? Dims - correction(Dims) : -1)...>; }; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/storage_info_rt.hpp0000644000175000017500000000343013613316060026165 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include #include "../../common/defs.hpp" namespace gridtools { /** \ingroup storage * @{ */ /* * @brief A storage info runtime object, providing dimensions and strides but without type information of the * storage info. Can be used in interfaces where no strict type information is available, i.e. in the interface to * Fortran. */ class storage_info_rt { using vec_t = std::vector; vec_t m_total_lengths; vec_t m_padded_lengths; vec_t m_strides; public: template storage_info_rt( TotalLengths const &total_lengths, PaddedLengths const &padded_lengths, Strides const &strides) { for (auto &&elem : total_lengths) m_total_lengths.push_back(elem); for (auto &&elem : padded_lengths) m_padded_lengths.push_back(elem); for (auto &&elem : strides) m_strides.push_back(elem); } const vec_t &total_lengths() const { return m_total_lengths; } const vec_t &padded_lengths() const { return m_padded_lengths; } const vec_t &strides() const { return m_strides; } }; /* * @brief Construct a storage_info_rt from a storage_info */ template storage_info_rt make_storage_info_rt(StorageInfo const &storage_info) { return {storage_info.total_lengths(), storage_info.padded_lengths(), storage_info.strides()}; } /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/storage_info.hpp0000644000175000017500000004000013613316060025452 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include #include #include "../../common/array.hpp" #include "../../common/array_dot_product.hpp" #include "../../common/defs.hpp" #include "../../common/error.hpp" #include "../../common/generic_metafunctions/accumulate.hpp" #include "../../common/generic_metafunctions/binary_ops.hpp" #include "../../common/generic_metafunctions/is_all_integrals.hpp" #include "../../common/host_device.hpp" #include "../../common/layout_map.hpp" #include "../../meta/type_traits.hpp" #include "alignment.hpp" #include "halo.hpp" #include "storage_info_metafunctions.hpp" namespace gridtools { /** \ingroup storage * @{ */ namespace impl_ { /** * @brief Internal helper function to check if two given storage infos contain the same information. * The function performs checks on all dimensions. This function is the base case. * @return true if the dimension, stride, size, initial_offset, etc. is equal, otherwise false */ template GT_FUNCTION std::enable_if_t equality_check(StorageInfo a, StorageInfo b) { return (a.template total_length() == b.template total_length()) && (a.template stride() == b.template stride()) && (a.length() == b.length()) && (a.total_length() == b.total_length()) && (a.padded_total_length() == b.padded_total_length()); } /** * @brief Internal helper function to check if two given storage infos contain the same information. * The function performs checks on all dimensions. This function is the step case. * @return true if the dimension, stride, size, initial_offset, etc. is equal, otherwise false */ template GT_FUNCTION std::enable_if_t equality_check(StorageInfo a, StorageInfo b) { return (a.template total_length() == b.template total_length()) && (a.template stride() == b.template stride()) && equality_check(a, b); } } // namespace impl_ /** * @brief The storage info interface. This class contains methods that should be implemented by all storage_info * implementations. * @tparam Id unique ID that should be shared among all storage infos with the same dimensionality. * @tparam Layout information about the memory layout * @tparam Halo information about the halo sizes (by default no halo is set) * @tparam Alignment information about the alignment */ template , typename Alignment = alignment<1>> struct storage_info; template struct storage_info, halo, Align> { using layout_t = layout_map; using halo_t = halo; using alignment_t = Align; static const int max_layout_v = layout_t::max(); GT_STATIC_ASSERT((sizeof...(Halos) == layout_t::masked_length), GT_INTERNAL_ERROR_MSG("Halo size does not match number of dimensions")); GT_STATIC_ASSERT(is_alignment::value, GT_INTERNAL_ERROR_MSG("Given type is not an alignment type")); static constexpr uint_t ndims = layout_t::masked_length; private: using this_t = storage_info, halo, Align>; array m_total_lengths; array m_padded_lengths; array m_strides; /* When computing the size of a storage, either length, total_length, or padded_total_length, we need to multiply the dimensions for those dimensions that are not associated to a -1 (masked-dimension). In addition when computing the size of the inner region, we need to remove the halos from the sizes. */ template > GT_FUNCTION static constexpr uint_t multiply_if_layout( std::integer_sequence, Array const &array, Halo h = zero_halo{}) { return accumulate( multiplies(), ((layout_t::template at() >= 0) ? array[Idxs] - 2 * h.at(Idxs) : 1)...); } template GT_FUNCTION GT_CONSTEXPR int offset(std::integer_sequence, Ints... idx) const { return accumulate(plus_functor(), (idx * m_strides[Seq])...); } template GT_FUNCTION GT_CONSTEXPR int first_index_impl(std::integer_sequence) const { return index(halo_t::template at()...); } template GT_FUNCTION GT_CONSTEXPR bool check_bounds(std::integer_sequence, Coords... coords) const { return accumulate(logical_and(), true, ((layout_t::template at() < 0) or (((int)coords >= 0) and (coords < m_total_lengths[Ints])))...); } public: constexpr static uint_t id = Id; /** * @brief storage info constructor. Additionally to initializing the members the halo * region is added to the corresponding dimensions and the alignment is applied. */ template ::value, int> = 0> GT_FUNCTION constexpr storage_info(Dims... dims_) : m_total_lengths{static_cast(dims_)...}, m_padded_lengths{pad_dimensions( handle_masked_dims::extend(dims_))...}, m_strides(get_strides::get_stride_array(pad_dimensions( handle_masked_dims::extend(dims_))...)) {} GT_FUNCTION storage_info(array const &dims, array const &strides) : m_total_lengths{dims}, m_strides(strides) { // We guess the padded lengths from the dimensions and the strides. Assume, that the strides are sorted, // e.g., [1, 16, 256], and the dimensions are [5, 9, 9]. For the largest stride (256), we assume that // padding = dimension (e.g. in this example the j-padding is 5). For all others we can calculate the // padding from the strides (e.g. in this example, the i-padding is 256 / 16 = 16, and the k-padding is 16 / // 1 = 16). Note that there might be strides which are set to 0 (masked dimensions). // // We first create a sorted copy of this array. We then loop over the unsorted array and set the padded // length for each entry as follows: // - If the stride is masked, the padded length is 0. // - If the stride is the maximum stride (i.e., 256 in the example above), the padding is derived from the // dimension. // - Otherwise, we find the stride s in the sorted array and we look for the next larger stride l in the // sorted array. The padded length is then set to l / s. Note that strides might appear several times. array sorted_strides; #pragma unroll for (uint_t i = 0; i < ndims; ++i) sorted_strides[i] = strides[i]; #pragma unroll for (uint_t i = 0; i < ndims; ++i) #pragma unroll for (uint_t j = 0; j < ndims; ++j) if (j > i && sorted_strides[i] > sorted_strides[j]) { auto tmp = sorted_strides[i]; sorted_strides[i] = sorted_strides[j]; sorted_strides[j] = tmp; } #pragma unroll for (uint_t i = 0; i < ndims; ++i) { if (strides[i] == sorted_strides[ndims - 1]) m_padded_lengths[i] = dims[i]; else if (strides[i] == 0) { m_padded_lengths[i] = 0; } else { // take the last stride that matches the stride we are looking for uint_t next_bigger_stride; #pragma unroll for (uint_t ii = 0; ii < ndims - 1; ++ii) if (strides[i] == sorted_strides[ii]) next_bigger_stride = sorted_strides[ii + 1]; m_padded_lengths[i] = next_bigger_stride / strides[i]; } } } /** * @brief storage info copy constructor. */ storage_info(storage_info const &other) = default; /** * @brief member function to retrieve the total size (dimensions, halos, initial_offset, padding). * @return total size including dimensions, halos, initial_offset, padding, and initial_offset */ GT_FUNCTION GT_CONSTEXPR uint_t padded_total_length() const { return multiply_if_layout(std::make_integer_sequence{}, m_padded_lengths); } /** * @brief member function to retrieve the number of domain elements * (dimensions, halos, no initial_offset, no padding). * @return number of domain elements */ GT_FUNCTION GT_CONSTEXPR uint_t total_length() const { return multiply_if_layout(std::make_integer_sequence{}, m_total_lengths); } /** * @brief member function to retrieve the number of inner domain elements * (dimensions, no halos, no initial_offset, no padding). * @return number of inner domain elements */ GT_FUNCTION GT_CONSTEXPR uint_t length() const { return multiply_if_layout(std::make_integer_sequence{}, m_total_lengths, halo_t{}); } /** * @brief Returns the array of total_lengths, the lengths including the halo points (the outer region) */ GT_FUNCTION GT_CONSTEXPR array const &total_lengths() const { return m_total_lengths; } /* * @brief Returns the length of a dimension including the halo points (the outer region) * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR int total_length() const { GT_STATIC_ASSERT( (Dim < ndims), GT_INTERNAL_ERROR_MSG("Out of bounds access in storage info dimension call.")); return m_total_lengths[Dim]; } /** * @brief Returns the length of a dimension including the halo points (the outer region) and padding. * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t padded_length() const { return m_padded_lengths[Dim]; } /** * @brief Returns the array of padded_lengths, the lengths including the halo points (the outer region) and * padding. */ GT_FUNCTION GT_CONSTEXPR array const &padded_lengths() const { return m_padded_lengths; } /** * @brief Returns the length of a dimension excluding the halo points (only the inner region) * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t length() const { return m_total_lengths[Dim] - 2 * halo_t::template at(); } /** * @brief Returns the index of the first element in the specified dimension when iterating in the whole outer * region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t total_begin() const { return 0; } /** * @brief Returns the index of the last element in the specified dimension when iterating in the whole outer * region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t total_end() const { return total_length() - 1; } /** * @brief Returns the index of the first element in the specified dimension when iterating in the inner region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t begin() const { return halo_t::template at(); } /** * @brief Returns the index of the last element in the specified dimension when iterating in the inner region * * \tparam Dim The index of the dimension */ template GT_FUNCTION GT_CONSTEXPR uint_t end() const { return begin() + length() - 1; } /** * @brief member function to retrieve the (aligned) stride (e.g., I, J, or K) * @tparam Coord queried coordinate * @return aligned stride size */ template GT_FUNCTION GT_CONSTEXPR uint_t stride() const { GT_STATIC_ASSERT((Dim < ndims), GT_INTERNAL_ERROR_MSG("Out of bounds access in storage info stride call.")); return get(m_strides); } /** * @brief return the array of (aligned) strides, see stride() for details. */ GT_FUNCTION GT_CONSTEXPR array const &strides() const { return m_strides; } /** * @brief member function to retrieve an offset (or index) when given offsets in I,J,K, etc. * E.g., index(1,2,3) --> 1*strideI + 2*strideJ + 3*strideK + initial_offset * @param idx given offsets * @return index */ template ::value, int> = 0> GT_FUNCTION GT_CONSTEXPR int index(Ints... idx) const { #ifdef NDEBUG return offset(std::make_integer_sequence{}, idx...); #else return error_or_return(check_bounds(std::make_integer_sequence{}, idx...), offset(std::make_integer_sequence{}, idx...), "Storage out of bounds access"); #endif } /** * @brief member function to retrieve an offset (or index) when given an array of offsets in I,J,K, etc. * E.g., index(1,2,3) --> 1*strideI + 2*strideJ + 3*strideK + initial_offset * @param offsets given offset array * @return index */ GT_FUNCTION GT_CONSTEXPR int index(gridtools::array const &offsets) const { return array_dot_product(offsets, m_strides); } GT_FUNCTION GT_CONSTEXPR int first_index_of_inner_region() const { return first_index_impl(std::make_integer_sequence{}); } /** * @brief function to check for equality of two given storage_infos * @param rhs right hand side storage info instance * @return true if the storage infos are equal, false otherwise */ GT_FUNCTION bool operator==(storage_info const &rhs) const { return impl_::equality_check(*this, rhs); } GT_FUNCTION bool operator!=(storage_info const &rhs) const { return !operator==(rhs); } }; template struct is_storage_info : std::false_type {}; template struct is_storage_info> : std::true_type {}; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/storage_info_metafunctions.hpp0000644000175000017500000000657413613316060030433 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include "../../common/array.hpp" #include "../../common/generic_metafunctions/is_all_integrals.hpp" #include "../../common/gt_assert.hpp" #include "../../common/layout_map.hpp" #include "alignment.hpp" #include "halo.hpp" namespace gridtools { /** \ingroup storage * @{ */ /* forward declaration */ template struct is_alignment; /* * @brief struct used to replace masked dimensions with size 1. otherwise the * passed dimension is returned unmodified. * @tparam LayoutArg layout map entry */ template struct handle_masked_dims { template GT_FUNCTION static constexpr uint_t extend(Dim d) { GT_STATIC_ASSERT( std::is_integral::value, GT_INTERNAL_ERROR_MSG("Dimensions has to be integral type.")); return error_or_return((d > 0), ((LayoutArg == -1) ? 1 : d), "Tried to instantiate storage info with zero or negative dimensions"); } }; /* * @brief function used to provide an aligned dimension * @tparam Alignment alignment information * @tparam Length Layout map length * @tparam LayoutArg layout map entry * @return return aligned dimension if it should be aligned, otherwise return as is. */ template GT_FUNCTION constexpr uint_t pad_dimensions(Int dimension) { GT_STATIC_ASSERT(is_alignment::value, GT_INTERNAL_ERROR_MSG("Passed type is no alignment type")); return ((Alignment::value > 1) && (LayoutArg == MaxLayoutV)) ? (dimension + Alignment::value - 1) / Alignment::value * Alignment::value : dimension; } /* * @brief struct used to compute the strides given the dimensions (e.g., 128x128x80) * @tparam Layout layout map */ template struct get_strides; template struct get_strides> { private: using layout_map_t = layout_map; template GT_FUNCTION static constexpr std::enable_if_t get_stride(Dims... d) { return (get_value_from_pack(layout_map_t::template find(), d...)) * get_stride(d...); } template GT_FUNCTION static constexpr std::enable_if_t get_stride(Dims...) { return 0; } template GT_FUNCTION static constexpr std::enable_if_t get_stride(Dims...) { return 1; } public: template ::value, int> = 0> GT_FUNCTION static constexpr array get_stride_array(Dims... ds) { return {get_stride(ds...)...}; } }; /** * @} */ } // namespace gridtools gridtools-1.1.3/include/gridtools/storage/common/halo.hpp0000644000175000017500000000532513613316060023731 0ustar alastairalastair/* * GridTools * * Copyright (c) 2014-2019, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. * SPDX-License-Identifier: BSD-3-Clause */ #pragma once #include #include "../../common/gt_assert.hpp" #include "../../common/variadic_pack_metafunctions.hpp" #include "../../meta/repeat.hpp" namespace gridtools { /** \ingroup storage * @{ */ /** * @brief A class that is used to pass halo information to the storage info. * E.g., Lets say we want to retrieve a storage_info instance with a halo * of 2 in I and J direction. We have to pass following type to the storage_facility * halo<2,2,0>. The I and J dimensions of the storage will be extended by 2 * in + and - direction. * @tparam N variadic list of halo sizes */ template struct halo { /** * @brief member function used to query the halo size of a given dimension * @tparam V Dimension or coordinate to query * @return halo size */ template GT_FUNCTION static constexpr uint_t at() { GT_STATIC_ASSERT( (V < sizeof...(N)), GT_INTERNAL_ERROR_MSG("Out of bounds access in halo type discovered.")); return get_value_from_pack(V, N...); } /** * @brief member function used to query the halo size of a given dimension * @param V Dimension or coordinate to query * @return halo size */ GT_FUNCTION static constexpr uint_t at(uint_t V) { return get_value_from_pack(V, N...); } /** * @brief member function used to query the number of dimensions. E.g., a halo * type with 3 entries cannot be passed to a <3 or >3 dimensional storage_info. * @return number of dimensions */ GT_FUNCTION static GT_CONSTEXPR uint_t size() { return sizeof...(N); } }; namespace _impl { template struct list_to_halo; template