pax_global_header00006660000000000000000000000064135724700560014523gustar00rootroot0000000000000052 comment=0a02ef603df8005e301df7547fbdf924d859c173 SpFFT-0.9.10/000077500000000000000000000000001357247005600125345ustar00rootroot00000000000000SpFFT-0.9.10/.clang-format000066400000000000000000000000771357247005600151130ustar00rootroot00000000000000BasedOnStyle: Google ColumnLimit: 100 AccessModifierOffset: -2 SpFFT-0.9.10/CI/000077500000000000000000000000001357247005600130275ustar00rootroot00000000000000SpFFT-0.9.10/CI/azure_templates/000077500000000000000000000000001357247005600162335ustar00rootroot00000000000000SpFFT-0.9.10/CI/azure_templates/ubuntu_build_test.yml000066400000000000000000000037271357247005600225270ustar00rootroot00000000000000parameters: # job config job_name: 'Ubuntu_GCC' run_mpi_tests: false run_local_tests: false #cmake parameters cm_mpi: 'ON' cm_omp: 'ON' cm_single_precision: 'OFF' cm_gpu_backend: 'OFF' jobs: - job: ${{ parameters.job_name }} pool: vmImage: 'ubuntu-latest' container: adhocman/master:ubuntu_18_04_cuda_9 strategy: matrix: Release: BUILD_TYPE: 'RELEASE' Debug: BUILD_TYPE: 'DEBUG' maxParallel: 2 variables: SPFFT_MPI: ${{ parameters.cm_mpi }} SPFFT_OMP: ${{ parameters.cm_omp }} SPFFT_SINGLE_PRECISION: ${{ parameters.cm_single_precision }} SPFFT_GPU_BACKEND: ${{ parameters.cm_gpu_backend }} steps: - script: | mkdir -p build cd build CUDACXX=/usr/local/cuda-9.0/bin/nvcc /opt/cmake-3.11.0/bin/cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DSPFFT_BUILD_TESTS=OFF -DSPFFT_MPI=${SPFFT_MPI} -DSPFFT_OMP=${SPFFT_OMP} -DSPFFT_SINGLE_PRECISION=${SPFFT_SINGLE_PRECISION} -DSPFFT_GPU_BACKEND=${SPFFT_GPU_BACKEND} make VERBOSE=1 displayName: 'Build library' - script: | cd build mkdir -p install_dir INSTALL_DIR=$(pwd)/install_dir /opt/cmake-3.11.0/bin/cmake .. -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} make VERBOSE=1 install test -f ${INSTALL_DIR}/lib/libspfft.so test -f ${INSTALL_DIR}/include/spfft/spfft.hpp test -f ${INSTALL_DIR}/include/spfft/spfft.h displayName: 'Install library' - script: | mkdir -p build cd build /opt/cmake-3.11.0/bin/cmake .. -DSPFFT_BUILD_TESTS=ON make VERBOSE=1 displayName: 'Build tests' - script: | export OMP_NUM_THREADS=2 ./build/tests/run_local_tests displayName: 'Run local tests' condition: and(succeeded(), eq('${{ parameters.run_local_tests }}', 'true')) - script: | export OMP_NUM_THREADS=2 mpirun -n 2 ./build/tests/run_mpi_tests displayName: 'Run MPI tests' condition: and(succeeded(), eq('${{ parameters.run_mpi_tests }}', 'true')) SpFFT-0.9.10/CMakeLists.txt000066400000000000000000000143751357247005600153060ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.11 FATAL_ERROR) # 3.11 to avoid issues with OpenMP + CUDA project(SpFFT LANGUAGES CXX VERSION 0.9.10) set(SPFFT_SO_VERSION 0) set(SPFFT_VERSION ${PROJECT_VERSION}) # allow {module}_ROOT variables to be set if(POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif() # use INTERFACE_LINK_LIBRARIES property if available if(POLICY CMP0022) cmake_policy(SET CMP0022 NEW) endif() # set default build type to RELEASE if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type" FORCE) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo" ) endif() # set language and standard set(CMAKE_CXX_STANDARD 11) #add local module path set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake/modules) # Options option(SPFFT_STATIC "Compile as static library" OFF) option(SPFFT_OMP "Compile with OpenMP support" ON) option(SPFFT_MPI "Compile with MPI support" ON) option(SPFFT_GPU_DIRECT "Compile with GPU direct (GPU aware MPI) support." OFF) option(SPFFT_BUILD_TESTS "Build tests" OFF) option(SPFFT_SINGLE_PRECISION "Enable single precision support" OFF) option(SPFFT_INSTALL "Enable CMake install commands" ON) option(SPFFT_FORTRAN "Compile fortran module" OFF) set(SPFFT_GPU_BACKEND "OFF" CACHE STRING "GPU backend") set_property(CACHE SPFFT_GPU_BACKEND PROPERTY STRINGS "OFF" "CUDA" "ROCM" ) # Get GNU standard install prefixes include(GNUInstallDirs) # set preferred library type if (SPFFT_STATIC) # prefer static over dynamic libraries with the find_library() command by changing the order set(CMAKE_FIND_LIBRARY_SUFFIXES_SAVE ${CMAKE_FIND_LIBRARY_SUFFIXES}) if(APPLE) set(CMAKE_FIND_LIBRARY_SUFFIXES .a .tbd .dylib .so) elseif(UNIX) set(CMAKE_FIND_LIBRARY_SUFFIXES .a .so) endif() set(SPFFT_LIBRARY_TYPE STATIC) else() set(SPFFT_LIBRARY_TYPE SHARED) endif() set(SPFFT_DEFINITIONS) set(SPFFT_EXTERNAL_COMPILE_OPTIONS) set(SPFFT_LIBS) set(SPFFT_EXTERNAL_LIBS) set(SPFFT_INTERFACE_LIBS) set(SPFFT_INTERFACE_INCLUDE_DIRS) set(SPFFT_INCLUDE_DIRS) set(SPFFT_EXTERNAL_INCLUDE_DIRS) set(SPFFT_EXTERNAL_PKG_PACKAGES) # Options combination check set(SPFFT_CUDA OFF) set(SPFFT_ROCM OFF) if(SPFFT_GPU_BACKEND) if(SPFFT_GPU_BACKEND STREQUAL "CUDA") set(SPFFT_CUDA ON) elseif(SPFFT_GPU_BACKEND STREQUAL "ROCM") set(SPFFT_ROCM ON) else() message(FATAL_ERROR "Invalid GPU backend option") endif() endif() mark_as_advanced(SPFFT_CUDA SPFFT_ROCM) # Hide symbols by default if tests are not build if(NOT SPFFT_BUILD_TESTS) set(CMAKE_CXX_VISIBILITY_PRESET hidden) set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) else() # disable visibility macros if tests are build list(APPEND SPFFT_DEFINITIONS -DSPFFT_STATIC_DEFINE) set(CMAKE_CXX_VISIBILITY_PRESET default) set(CMAKE_VISIBILITY_INLINES_HIDDEN 0) endif() # Fortran if(SPFFT_FORTRAN) enable_language(Fortran) endif() # CUDA if(SPFFT_CUDA) enable_language(CUDA) find_library(CUDA_CUDART_LIBRARY cudart PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) find_library(CUDA_CUFFT_LIBRARY cufft PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) list(APPEND SPFFT_EXTERNAL_LIBS ${CUDA_CUDART_LIBRARY} ${CUDA_CUFFT_LIBRARY}) list(APPEND SPFFT_EXTERNAL_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) endif() # ROCM if(SPFFT_ROCM) find_package(ROCM REQUIRED COMPONENTS rocfft) list(APPEND SPFFT_EXTERNAL_INCLUDE_DIRS ${ROCM_INCLUDE_DIRS}) list(APPEND SPFFT_EXTERNAL_LIBS ${ROCM_LIBRARIES}) list(APPEND SPFFT_EXTERNAL_COMPILE_OPTIONS ${ROCM_DEFINITIONS}) endif() if(SPFFT_MPI) find_package(MPI COMPONENTS CXX REQUIRED) list(APPEND SPFFT_EXTERNAL_LIBS MPI::MPI_CXX) # always add MPI to interface libraries, because mpi.h is included in public header files if(SPFFT_STATIC) list(APPEND SPFFT_INTERFACE_LIBS ${MPI_CXX_LIBRARIES}) endif() list(APPEND SPFFT_INTERFACE_INCLUDE_DIRS ${MPI_CXX_INCLUDE_DIRS}) endif() if(SPFFT_OMP) find_package(OpenMP COMPONENTS CXX REQUIRED) list(APPEND SPFFT_EXTERNAL_LIBS OpenMP::OpenMP_CXX) if(SPFFT_STATIC) list(APPEND SPFFT_INTERFACE_LIBS ${OpenMP_CXX_LIBRARIES}) endif() endif() if(SPFFT_GPU_DIRECT) message(STATUS "GPU Direct support enabled: Additional environment variables might have to be set before execution. (e.g \"export MPICH_RDMA_ENABLED_CUDA=1\")") endif() # Use MKL if available, otherwise require FFTW3 if(UNIX AND NOT APPLE) # prefer static MKL in Linux. Together with "-Wl,--exclude-libs,ALL", # symbols are not visible for linking afterwards and no conflicts with other MKL versions of other libraries should exist. set(_TMP_SAVE ${CMAKE_FIND_LIBRARY_SUFFIXES}) set(CMAKE_FIND_LIBRARY_SUFFIXES .a .so) endif() find_package(MKLSequential) if(UNIX AND NOT APPLE) set(CMAKE_FIND_LIBRARY_SUFFIXES ${_TMP_SAVE}) unset(_TMP_SAVE) endif() if(MKLSequential_FOUND) list(APPEND SPFFT_EXTERNAL_LIBS MKL::Sequential) if(SPFFT_STATIC) list(APPEND SPFFT_INTERFACE_LIBS ${MKLSequential_LIBRARIES}) endif() list(APPEND SPFFT_EXTERNAL_PKG_PACKAGES mkl-dynamic-lp64-seq) else() find_package(FFTW REQUIRED) list(APPEND SPFFT_EXTERNAL_LIBS FFTW::FFTW) if(SPFFT_STATIC) list(APPEND SPFFT_INTERFACE_LIBS ${FFTW_LIBRARIES}) endif() if(SPFFT_SINGLE_PRECISION AND NOT FFTW_FLOAT_FOUND) message(FATAL_ERROR "FFTW library with single precision support NOT FOUND. Disable SPFFT_SINGLE_PRECISION or provide path to library.") endif() list(APPEND SPFFT_EXTERNAL_PKG_PACKAGES fftw3) endif() if(SPFFT_BUILD_TESTS) # enable timing with testing set(SPFFT_TIMING ON) endif() # generate config.h configure_file(include/spfft/config.h.in ${PROJECT_BINARY_DIR}/spfft/config.h) list(APPEND SPFFT_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/src) list(APPEND SPFFT_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/include) list(APPEND SPFFT_INCLUDE_DIRS ${PROJECT_BINARY_DIR}) list(APPEND SPFFT_EXTERNAL_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/ext) ############################################################################# # All include dirs and definitions must be set before sub-directory is added! ############################################################################# add_subdirectory(src) list(APPEND SPFFT_LIBS spfft) # add tests for developement if(SPFFT_BUILD_TESTS) add_subdirectory(tests) endif() # reset cmake library suffixes if(SPFFT_STATIC) set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAVE}) endif() SpFFT-0.9.10/LICENSE000066400000000000000000000027151357247005600135460ustar00rootroot00000000000000Copyright (c) 2019 ETH Zurich, Simon Frasch Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SpFFT-0.9.10/README.md000066400000000000000000000155601357247005600140220ustar00rootroot00000000000000[![Build Status](https://dev.azure.com/zeadreamplay/SpFFT/_apis/build/status/eth-cscs.SpFFT?branchName=master)](https://dev.azure.com/zeadreamplay/SpFFT/_build/latest?definitionId=3&branchName=master) [![Documentation](https://readthedocs.org/projects/spfft/badge/?version=latest)](https://spfft.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/eth-cscs/SpFFT/master/LICENSE) # SpFFT SpFFT - A 3D FFT library for sparse frequency domain data written in C++ with support for MPI, OpenMP, CUDA and ROCm. It was originally intended for transforms of data with spherical cutoff in frequency domain, as required by some computational material science codes. For distributed computations, SpFFT uses a slab decomposition in space domain and pencil decomposition in frequency domain (all sparse data within a pencil must be on one rank). ***Fig. 1:*** Illustration of a transform, where data on each MPI rank is identified by color. ### Design Goals - Sparse frequency domain input - Reuse of pre-allocated memory - Support of negative indexing for frequency domain data - Unified interface for calculations on CPUs and GPUs - Support of Complex-To-Real and Real-To-Complex transforms, where the full hermitian symmetry property is utilized. - C++, C and Fortran interfaces - Parallelization and acceleration are optional ### Interface Design To allow for pre-allocation and reuse of memory, the design is based on two classes: - **Grid**: Allocates memory for transforms up to a given size in each dimension. - **Transform**: Is associated with a *Grid* and can have any size up to the *Grid* dimensions. A *Transform* holds a counted reference to the underlying *Grid*. Therefore, *Transforms* created with the same *Grid* share memory, which is only freed, once the *Grid* and all associated *Transforms* are destroyed. The user provides memory for storing sparse frequency domain data, while a *Transform* provides memory for space domain data. This implies, that executing a *Transform* will override the space domain data of all other *Transforms* associated with the same *Grid*. ## Documentation Documentation can be found [here](https://spfft.readthedocs.io/en/latest/). ## Requirements - C++ Compiler with C++11 support. Supported compilers are: - GCC 6 and later - Clang 5 and later - ICC 18.0 and later - CMake 3.11 and later - Library providing a FFTW 3.x interface (FFTW3 or Intel MKL) - For multi-threading: OpenMP support by the compiler - For compilation with GPU support: - CUDA 9.0 and later for Nvidia hardware - ROCm 2.6 and later for AMD hardware ## Installation The build system follows the standard CMake workflow. Example: ```console mkdir build cd build cmake .. -DSPFFT_OMP=ON -DSPFFT_MPI=ON -DSPFFT_GPU_BACKEND=CUDA -DSPFFT_SINGLE_PRECISION=OFF -DCMAKE_INSTALL_PREFIX=/usr/local make -j8 install ``` ### CMake options | Option | Default | Description | |------------------------|---------|--------------------------------------------------| | SPFFT_MPI | ON | Enable MPI support | | SPFFT_OMP | ON | Enable multi-threading with OpenMP | | SPFFT_GPU_BACKEND | OFF | Select GPU backend. Can be OFF, CUDA or ROCM | | SPFFT_GPU_DIRECT | OFF | Use GPU aware MPI with GPUDirect | | SPFFT_SINGLE_PRECISION | OFF | Enable single precision support | | SPFFT_STATIC | OFF | Build as static library | | SPFFT_BUILD_TESTS | OFF | Build test executables for developement purposes | | SPFFT_INSTALL | ON | Add library to install target | | SPFFT_FORTRAN | OFF | Build Fortran interface module | ## Examples Further exmples for C++, C and Fortran can be found in the "examples" folder. ```cpp #include #include #include #include "spfft/spfft.hpp" int main(int argc, char** argv) { const int dimX = 2; const int dimY = 2; const int dimZ = 2; std::cout << "Dimensions: x = " << dimX << ", y = " << dimY << ", z = " << dimZ << std::endl << std::endl; // Use default OpenMP value const int numThreads = -1; // use all elements in this example. const int numFrequencyElements = dimX * dimY * dimZ; // Slice length in space domain. Equivalent to dimZ for non-distributed case. const int localZLength = dimZ; // interleaved complex numbers std::vector frequencyElements; frequencyElements.reserve(2 * numFrequencyElements); // indices of frequency elements std::vector indices; indices.reserve(dimX * dimY * dimZ * 3); // initialize frequency domain values and indices double initValue = 0.0; for (int xIndex = 0; xIndex < dimX; ++xIndex) { for (int yIndex = 0; yIndex < dimY; ++yIndex) { for (int zIndex = 0; zIndex < dimZ; ++zIndex) { // init with interleaved complex numbers frequencyElements.emplace_back(initValue); frequencyElements.emplace_back(-initValue); // add index triplet for value indices.emplace_back(xIndex); indices.emplace_back(yIndex); indices.emplace_back(zIndex); initValue += 1.0; } } } std::cout << "Input:" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } // create local Grid. For distributed computations, a MPI Communicator has to be provided spfft::Grid grid(dimX, dimY, dimZ, dimX * dimY, SPFFT_PU_HOST, numThreads); // create transform spfft::Transform transform = grid.create_transform(SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, localZLength, numFrequencyElements, SPFFT_INDEX_TRIPLETS, indices.data()); // Get pointer to space domain data. Alignment fullfills requirements for std::complex. // Can also be read as std::complex elements (guaranteed by the standard to be binary compatible // since C++11). double* spaceDomain = transform.space_domain_data(SPFFT_PU_HOST); // transform backward transform.backward(frequencyElements.data(), SPFFT_PU_HOST); std::cout << std::endl << "After backward transform:" << std::endl; for (int i = 0; i < transform.local_slice_size(); ++i) { std::cout << spaceDomain[2 * i] << ", " << spaceDomain[2 * i + 1] << std::endl; } // transform forward transform.forward(SPFFT_PU_HOST, frequencyElements.data(), SPFFT_NO_SCALING); std::cout << std::endl << "After forward transform (without scaling):" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } return 0; } ``` SpFFT-0.9.10/azure-pipelines.yml000066400000000000000000000032541357247005600163770ustar00rootroot00000000000000####################### # Linux ####################### jobs: # CUDA build - template: CI/azure_templates/ubuntu_build_test.yml parameters: job_name: "Ubuntu_GCC_CUDA" run_mpi_tests: false run_local_tests: false cm_mpi: 'ON' cm_omp: 'ON' cm_single_precision: 'ON' cm_gpu_backend: 'CUDA' # buld and test without MPI and OpenMP - template: CI/azure_templates/ubuntu_build_test.yml parameters: job_name: "Ubuntu_GCC_no_parallel" run_mpi_tests: false run_local_tests: true cm_mpi: 'OFF' cm_omp: 'OFF' cm_single_precision: 'OFF' cm_gpu_backend: 'OFF' # buld and test with full parallelization - template: CI/azure_templates/ubuntu_build_test.yml parameters: job_name: "Ubuntu_GCC_parallel" run_mpi_tests: true run_local_tests: true cm_mpi: 'ON' cm_omp: 'ON' cm_single_precision: 'OFF' cm_gpu_backend: 'OFF' ####################### # macOS ####################### - job: macOS_Clang pool: vmImage: 'macOS-10.14' steps: - script: | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" displayName: 'Install Homebrew' - script: | brew install fftw brew install open-mpi displayName: 'Install dependencies' # Apple Clang does not support OpenMP - script: | mkdir -p build cd build cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DSPFFT_BUILD_TESTS=ON -DSPFFT_MPI=ON -DSPFFT_OMP=OFF make VERBOSE=1 displayName: 'Build' - script: | ./build/tests/run_local_tests displayName: 'Run local tests' - script: | mpirun -n 2 ./build/tests/run_mpi_tests displayName: 'Run MPI tests' SpFFT-0.9.10/cmake/000077500000000000000000000000001357247005600136145ustar00rootroot00000000000000SpFFT-0.9.10/cmake/SpFFT.pc.in000066400000000000000000000005361357247005600154730ustar00rootroot00000000000000prefix=@CMAKE_INSTALL_PREFIX@ libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ external_packages= Name: SpFFT Description: Sparse 3D FFT library with MPI, OpenMP, CUDA and ROCm support Version: @PROJECT_VERSION@ Libs: -L${libdir} -lspfft Cflags: -I${includedir} Requires.private: @SPFFT_EXTERNAL_PKG_PACKAGES@ SpFFT-0.9.10/cmake/SpFFTConfig.cmake000066400000000000000000000003131357247005600166630ustar00rootroot00000000000000 if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedConfig.cmake") include("${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedConfig.cmake") else() include("${CMAKE_CURRENT_LIST_DIR}/SpFFTStaticConfig.cmake") endif() SpFFT-0.9.10/cmake/SpFFTConfigVersion.cmake000066400000000000000000000003701357247005600202340ustar00rootroot00000000000000 # Prefer shared library if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedConfigVersion.cmake") include("${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedConfigVersion.cmake") else() include("${CMAKE_CURRENT_LIST_DIR}/SpFFTStaticConfigVersion.cmake") endif() SpFFT-0.9.10/cmake/SpFFTSharedConfig.cmake000066400000000000000000000006671357247005600200260ustar00rootroot00000000000000# options used for building library set(SPFFT_OMP @SPFFT_OMP@) set(SPFFT_MPI @SPFFT_MPI@) set(SPFFT_STATIC @SPFFT_STATIC@) set(SPFFT_GPU_DIRECT @SPFFT_GPU_DIRECT@) set(SPFFT_SINGLE_PRECISION @SPFFT_SINGLE_PRECISION@) set(SPFFT_GPU_BACKEND @SPFFT_GPU_BACKEND@) # add version of package include("${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedConfigVersion.cmake") # add library target include("${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedTargets.cmake") SpFFT-0.9.10/cmake/SpFFTStaticConfig.cmake000066400000000000000000000006671357247005600200470ustar00rootroot00000000000000# options used for building library set(SPFFT_OMP @SPFFT_OMP@) set(SPFFT_MPI @SPFFT_MPI@) set(SPFFT_STATIC @SPFFT_STATIC@) set(SPFFT_GPU_DIRECT @SPFFT_GPU_DIRECT@) set(SPFFT_SINGLE_PRECISION @SPFFT_SINGLE_PRECISION@) set(SPFFT_GPU_BACKEND @SPFFT_GPU_BACKEND@) # add version of package include("${CMAKE_CURRENT_LIST_DIR}/SpFFTStaticConfigVersion.cmake") # add library target include("${CMAKE_CURRENT_LIST_DIR}/SpFFTStaticTargets.cmake") SpFFT-0.9.10/cmake/SpFFTTargets.cmake000066400000000000000000000003461357247005600170750ustar00rootroot00000000000000 # Prefer shared library if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedTargets.cmake") include("${CMAKE_CURRENT_LIST_DIR}/SpFFTSharedTargets.cmake") else() include("${CMAKE_CURRENT_LIST_DIR}/SpFFTStaticTargets.cmake") endif() SpFFT-0.9.10/cmake/modules/000077500000000000000000000000001357247005600152645ustar00rootroot00000000000000SpFFT-0.9.10/cmake/modules/FindFFTW.cmake000066400000000000000000000073621357247005600176450ustar00rootroot00000000000000# Copyright (c) 2019 ETH Zurich, Simon Frasch # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of the copyright holder nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. #.rst: # FindFFTW # ----------- # # This module searches for the fftw3 library. # # The following variables are set # # :: # # FFTW_FOUND - True if double precision fftw library is found # FFTW_FLOAT_FOUND - True if single precision fftw library is found # FFTW_LIBRARIES - The required libraries # FFTW_INCLUDE_DIRS - The required include directory # # The following import target is created # # :: # # FFTW::FFTW # set paths to look for library set(_FFTW_PATHS ${FFTW_ROOT} $ENV{FFTW_ROOT}) set(_FFTW_INCLUDE_PATHS) set(_FFTW_DEFAULT_PATH_SWITCH) if(_FFTW_PATHS) # disable default paths if ROOT is set set(_FFTW_DEFAULT_PATH_SWITCH NO_DEFAULT_PATH) else() # try to detect location with pkgconfig find_package(PkgConfig QUIET) if(PKG_CONFIG_FOUND) pkg_check_modules(PKG_FFTW QUIET "fftw3") endif() set(_FFTW_PATHS ${PKG_FFTW_LIBRARY_DIRS}) set(_FFTW_INCLUDE_PATHS ${PKG_FFTW_INCLUDE_DIRS}) endif() find_library( FFTW_LIBRARIES NAMES "fftw3" HINTS ${_FFTW_PATHS} PATH_SUFFIXES "lib" "lib64" ${_FFTW_DEFAULT_PATH_SWITCH} ) find_library( _FFTW_FLOAT_LIBRARY NAMES "fftw3f" HINTS ${_FFTW_PATHS} PATH_SUFFIXES "lib" "lib64" ${_FFTW_DEFAULT_PATH_SWITCH} ) find_path(FFTW_INCLUDE_DIRS NAMES "fftw3.h" HINTS ${_FFTW_PATHS} ${_FFTW_INCLUDE_PATHS} PATH_SUFFIXES "include" "include/fftw" ${_FFTW_DEFAULT_PATH_SWITCH} ) # check if found include(FindPackageHandleStandardArgs) find_package_handle_standard_args(FFTW REQUIRED_VARS FFTW_INCLUDE_DIRS FFTW_LIBRARIES ) # check if single precision library found if(_FFTW_FLOAT_LIBRARY AND FFTW_FOUND) list(APPEND FFTW_LIBRARIES ${_FFTW_FLOAT_LIBRARY}) set(FFTW_FLOAT_FOUND TRUE) else() set(FFTW_FLOAT_FOUND FALSE) endif() # add target to link against if(FFTW_FOUND) if(NOT TARGET FFTW::FFTW) add_library(FFTW::FFTW INTERFACE IMPORTED) endif() set_property(TARGET FFTW::FFTW PROPERTY INTERFACE_LINK_LIBRARIES ${FFTW_LIBRARIES}) set_property(TARGET FFTW::FFTW PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${FFTW_INCLUDE_DIRS}) endif() # prevent clutter in cache MARK_AS_ADVANCED(FFTW_FOUND FFTW_LIBRARIES FFTW_INCLUDE_DIRS pkgcfg_lib_PKG_FFTW_fftw3 _FFTW_FLOAT_LIBRARY) SpFFT-0.9.10/cmake/modules/FindMKLSequential.cmake000066400000000000000000000131331357247005600215460ustar00rootroot00000000000000# Copyright (c) 2019 ETH Zurich, Simon Frasch # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. Neither the name of the copyright holder nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. #.rst: # FindMKLSequential # ----------- # # This module searches for the sequential 32-bit integer MKL library. # Only looks for static libraries by default. # # # The following variables are set # # :: # # MKLSequential_FOUND - True if double precision fftw library is found # MKLSequential_LIBRARIES - The required libraries # MKLSequential_INCLUDE_DIRS - The required include directory # MKLSequential_FFTW_INCLUDE_DIRS - The required fftw interface include directory # # The following import target is created # # :: # # MKL::Sequential # set paths to look for MKL set(_MKLSequential_PATHS ${MKLSequential_ROOT} $ENV{MKLROOT}) set(_MKLSequential_INCLUDE_PATHS) set(_MKLSequential_DEFAULT_PATH_SWITCH) if(_MKLSequential_PATHS) # do not look at any default paths if a custom path was set set(_MKLSequential_DEFAULT_PATH_SWITCH NO_DEFAULT_PATH) else() # try to detect location with pkgconfig if(NOT MKLSequential_ROOT) find_package(PkgConfig QUIET) if(PKG_CONFIG_FOUND) # look for dynmic module, such that a -L flag can be parsed pkg_check_modules(PKG_MKL QUIET "mkl-dynamic-lp64-seq") set(_MKLSequential_PATHS ${PKG_MKL_LIBRARY_DIRS}) set(_MKLSequential_INCLUDE_PATHS ${PKG_MKL_INCLUDE_DIRS}) endif() endif() endif() # find all MKL libraries / include directories find_library( _MKLSequential_INT_LIB NAMES "mkl_intel_lp64" HINTS ${_MKLSequential_PATHS} PATH_SUFFIXES "intel64_lin" "intel64" "lib/intel64_lin" "lib/intel64" ${_MKLSequential_DEFAULT_PATH_SWITCH} ) find_library( _MKLSequential_SEQ_LIB NAMES "mkl_sequential" HINTS ${_MKLSequential_PATHS} PATH_SUFFIXES "intel64_lin" "intel64" "lib/intel64_lin" "lib/intel64" ${_MKLSequential_DEFAULT_PATH_SWITCH} ) find_library( _MKLSequential_CORE_LIB NAMES "mkl_core" HINTS ${_MKLSequential_PATHS} PATH_SUFFIXES "intel64_lin" "intel64" "lib/intel64_lin" "lib/intel64" ${_MKLSequential_DEFAULT_PATH_SWITCH} ) find_path(MKLSequential_INCLUDE_DIRS NAMES "mkl.h" HINTS ${_MKLSequential_PATHS} ${_MKLSequential_INCLUDE_PATHS} PATH_SUFFIXES "include" ${_MKLSequential_DEFAULT_PATH_SWITCH} ) find_path(MKLSequential_FFTW_INCLUDE_DIRS NAMES "fftw3.h" HINTS ${_MKLSequential_PATHS} ${_MKLSequential_INCLUDE_PATHS} PATH_SUFFIXES "include" "include/fftw" "fftw" ${_MKLSequential_DEFAULT_PATH_SWITCH} ) # check if found include(FindPackageHandleStandardArgs) find_package_handle_standard_args(MKLSequential REQUIRED_VARS _MKLSequential_INT_LIB _MKLSequential_SEQ_LIB _MKLSequential_CORE_LIB MKLSequential_INCLUDE_DIRS MKLSequential_FFTW_INCLUDE_DIRS) # add target to link against if(MKLSequential_FOUND) # libries have inter-dependencies, therefore use link group on Linux if(UNIX AND NOT APPLE) set(MKLSequential_LIBRARIES "-Wl,--start-group" ${_MKLSequential_INT_LIB} ${_MKLSequential_SEQ_LIB} ${_MKLSequential_CORE_LIB} "-Wl,--end-group") else() set(MKLSequential_LIBRARIES ${_MKLSequential_INT_LIB} ${_MKLSequential_SEQ_LIB} ${_MKLSequential_CORE_LIB}) endif() # external libries required on unix if(UNIX) list(APPEND MKLSequential_LIBRARIES -lpthread -lm -ldl) endif() # create interface target if(NOT TARGET MKL::Sequential) add_library(MKL::Sequential INTERFACE IMPORTED) endif() set_property(TARGET MKL::Sequential PROPERTY INTERFACE_LINK_LIBRARIES ${MKLSequential_LIBRARIES}) set_property(TARGET MKL::Sequential PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MKLSequential_INCLUDE_DIRS} ${MKLSequential_FFTW_INCLUDE_DIRS}) endif() # prevent clutter in gui MARK_AS_ADVANCED(MKLSequential_FOUND MKLSequential_LIBRARIES MKLSequential_INCLUDE_DIRS _MKLSequential_INT_LIB _MKLSequential_SEQ_LIB _MKLSequential_CORE_LIB MKLSequential_FFTW_INCLUDE_DIRS _MKLSequential_DEFAULT_PATH_SWITCH _MKLSequential_PATHS) MARK_AS_ADVANCED(pkgcfg_lib_PKG_MKL_dl pkgcfg_lib_PKG_MKL_m pkgcfg_lib_PKG_MKL_mkl_core pkgcfg_lib_PKG_MKL_mkl_sequential pkgcfg_lib_PKG_MKL_mkl_intel_lp64 pkgcfg_lib_PKG_MKL_pthread) SpFFT-0.9.10/cmake/modules/FindROCM.cmake000066400000000000000000000466221357247005600176410ustar00rootroot00000000000000# - Find the ROCM library # # Usage: # find_package(ROCM [REQUIRED] [QUIET] COMPONENTS [components ...] ) # # Compnents available: # - hipblas # - hipsparse # - rocfft # - rocblas # - rocsparse # # Commands made available: # rocm_hip_add_library( [STATIC | SHARED] [FLAGS] [OUTPUT_DIR] [INCLUDE_DIRS] ) # --- Compiles source files into an imported library with hipcc. No global defitions or include directories are taken into account. # # The following variables can be set for compilation: # ROCM_HIPCC_FLAGS ----------------- Flags passed on to hipcc compiler # ROCM_HIPCC_FLAGS_DEBUG ----------- Flags passed on to hipcc compiler in DEBUG mode # ROCM_HIPCC_FLAGS_RELEASE --------- Flags passed on to hipcc compiler in RELEASE mode # ROCM_HIPCC_FLAGS_RELWITHDEBINFO -- Flags passed on to hipcc compiler in RELWITHDEBINFO mode # ROCM_HIPCC_FLAGS_MINSIZEREL ------ Flags passed on to hipcc compiler in MINSIZEREL mode # # The following variables can be set to specify a search location # ROCM_ROOT ------------ if set, the libraries are exclusively searched under this path # _ROOT ------ if set, search for component specific libraries at given path. Takes precedence over ROCM_ROOT # # The following variables are generated: # ROCM_FOUND ------------------- true if ROCM is found on the system # ROCM_LIBRARIES --------------- full path to ROCM # ROCM_INCLUDE_DIRS ------------ ROCM include directories # ROCM_DEFINITIONS ------------- ROCM definitions # ROCM_HCC_EXECUTABLE ---------- ROCM HCC compiler # ROCM_HCC-CONFIG_EXECUTABLE --- ROCM HCC config # ROCM_HIPCC_EXECUTABLE -------- HIPCC compiler # ROCM_HIPCONFIG_EXECUTABLE ---- hip config # ROCM_HIPIFY-PERL_EXECUTABLE -- hipify # ROCM_HIP_PLATFORM ------------ Platform identifier: "hcc" or "nvcc" # set(ROCM_HIPCC_FLAGS "" CACHE STRING "Flags for HIPCC Compiler") set(ROCM_HIPCC_FLAGS_DEBUG "-g" CACHE STRING "Debug flags for HIPCC Compiler") set(ROCM_HIPCC_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "Release flags for HIPCC Compiler") set(ROCM_HIPCC_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG" CACHE STRING "Release with debug flags for HIPCC Compiler") set(ROCM_HIPCC_FLAGS_MINSIZEREL "-Os -DNDEBUG" CACHE STRING "Minimum size flags for HIPCC Compiler") #If environment variable ROCM_ROOT is specified if(NOT ROCM_ROOT AND ENV{ROCM_ROOT}) file(TO_CMAKE_PATH "$ENV{ROCM_ROOT}" ROCM_ROOT) set(ROCM_ROOT "${ROCM_ROOT}" CACHE PATH "Root directory for ROCM installation.") endif() set(ROCM_FOUND FALSE) set(ROCM_LIBRARIES) set(ROCM_INCLUDE_DIRS) set(ROCM_DEFINITIONS) unset(ROCM_HCC_EXECUTABLE) unset(ROCM_HCC-CONFIG_EXECUTABLE) unset(ROCM_HIPCC_EXECUTABLE) unset(ROCM_HIPCONFIG_EXECUTABLE) unset(ROCM_HIPFIY-PERL-EXECUTABLE) unset(ROCM_HIP_PLATFORM) include(FindPackageHandleStandardArgs) # Finds libraries and include path for rocm modules # IN: # - module_name: name of a module (e.g. hcc) # - following arguments: name of libraries required # OUT: # - ROCM_LIBRARIES: Appends to list of libraries # - ROCM_INCLUDE_DIRS: Appends to include dirs function(find_rcm_module module_name) # convert module name to upper case for consistent variable naming string(TOUPPER ${module_name} MODULE_NAME_UPPER) if(DEFINED ${MODULE_NAME_UPPER}_ROOT) set(ROOT_DIR ${${MODULE_NAME_UPPER}_ROOT}) elseif(DEFINED ROCM_ROOT) set(ROOT_DIR ${ROCM_ROOT}) endif() # get abosolute path to avoid issues with tilde if(ROOT_DIR) get_filename_component(ROOT_DIR ${ROOT_DIR} ABSOLUTE) endif() # remove module name from input arguments set(LIBRARY_NAMES ${ARGV}) list(REMOVE_AT LIBRARY_NAMES 0) if(${ROCM_FIND_REQUIRED}) set(ROCM_${MODULE_NAME_UPPER}_FIND_REQUIRED TRUE) else() set(ROCM_${MODULE_NAME_UPPER}_FIND_REQUIRED FALSE) endif() if(${ROCM_FIND_QUIETLY}) set(ROCM_${MODULE_NAME_UPPER}_FIND_QUIETLY TRUE) else() set(ROCM_${MODULE_NAME_UPPER}_FIND_QUIETLY FALSE) endif() set(ROCM_LIBRARIES_${MODULE_NAME_UPPER}) if(ROOT_DIR) # find libraries foreach(library_name IN LISTS LIBRARY_NAMES) find_library( ROCM_LIBRARIES_${library_name} NAMES ${library_name} PATHS ${ROOT_DIR} PATH_SUFFIXES "lib" "${module_name}/lib" NO_DEFAULT_PATH ) find_package_handle_standard_args(ROCM_${MODULE_NAME_UPPER} FAIL_MESSAGE "For ROCM module ${module_name}, library ${library_name} could not be found. Please specify ROCM_ROOT or ${MODULE_NAME_UPPER}_ROOT." REQUIRED_VARS ROCM_LIBRARIES_${library_name}) if(ROCM_LIBRARIES_${library_name}) list(APPEND ROCM_LIBRARIES_${MODULE_NAME_UPPER} ${ROCM_LIBRARIES_${library_name}}) mark_as_advanced(ROCM_LIBRARIES_${library_name}) endif() endforeach() # find include directory find_path( ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER} NAMES ${module_name}/include PATHS ${ROOT_DIR} ${ROOT_DIR}/.. NO_DEFAULT_PATH ) # set include directory for module if found if(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) set(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER} ${ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}}/${module_name}/include) endif() else() foreach(library_name IN LISTS LIBRARY_NAMES) find_library( ROCM_LIBRARIES_${library_name} NAMES ${library_name} PATHS /opt/rocm PATH_SUFFIXES "lib" "lib64" "${module_name}/lib" "rocm/${module_name}/lib" ) find_package_handle_standard_args(ROCM_${MODULE_NAME_UPPER} FAIL_MESSAGE "For ROCM module ${module_name}, library ${library_name} could not be found. Please specify ROCM_ROOT or ${MODULE_NAME_UPPER}_ROOT." REQUIRED_VARS ROCM_LIBRARIES_${library_name}) if(ROCM_LIBRARIES_${library_name}) list(APPEND ROCM_LIBRARIES_${MODULE_NAME_UPPER} ${ROCM_LIBRARIES_${library_name}}) mark_as_advanced(ROCM_LIBRARIES_${library_name}) endif() endforeach() # find include directory find_path( ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER} NAMES ${module_name}/include PATHS /opt/rocm/ ) # set include directory for module if found if(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) set(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER} ${ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}}/${module_name}/include) endif() endif() # check if all required parts found find_package_handle_standard_args(ROCM_${MODULE_NAME_UPPER} FAIL_MESSAGE "ROCM module ${module_name} could not be found. Please specify ROCM_ROOT or ${MODULE_NAME_UPPER}_ROOT." REQUIRED_VARS ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) if(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) mark_as_advanced(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) endif() # set global variables if(ROCM_LIBRARIES_${MODULE_NAME_UPPER}) set(ROCM_LIBRARIES ${ROCM_LIBRARIES} ${ROCM_LIBRARIES_${MODULE_NAME_UPPER}} PARENT_SCOPE) endif() if(ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}) set(ROCM_INCLUDE_DIRS ${ROCM_INCLUDE_DIRS} ${ROCM_INCLUDE_DIRS_${MODULE_NAME_UPPER}} PARENT_SCOPE) endif() endfunction() # Finds executables of rocm modules # IN: # - module_name: name of a module (e.g. hcc) # - executable_name: name of the executable (e.g. hcc) # OUT: # - ROCM_${executable_name}_EXECUTABLE: Path to executable function(find_rocm_executable module_name executable_name) string(TOUPPER ${module_name} MODULE_NAME_UPPER) string(TOUPPER ${executable_name} EXECUTABLE_NAME_UPPER) unset(ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE PARENT_SCOPE) if(DEFINED ${MODULE_NAME_UPPER}_ROOT) set(ROOT_DIR ${${MODULE_NAME_UPPER}_ROOT}) elseif(DEFINED ROCM_ROOT) set(ROOT_DIR ${ROCM_ROOT}) endif() # get abosolute path to avoid issues with tilde if(ROOT_DIR) get_filename_component(ROOT_DIR ${ROOT_DIR} ABSOLUTE) endif() if(ROOT_DIR) find_file( ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE NAMES ${executable_name} PATHS ${ROOT_DIR} PATH_SUFFIXES "bin" "${module_name}/bin" NO_DEFAULT_PATH ) else() find_file( ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE NAMES ${executable_name} PATHS "/opt/rocm" PATH_SUFFIXES "bin" "${module_name}/bin" ) endif() set(ROCM_${EXECUTABLE_NAME_UPPER} ROCM_${EXECUTABLE_NAME_UPPER} PARENT_SCOPE) if(${ROCM_FIND_REQUIRED}) set(ROCM_${MODULE_NAME_UPPER}_${EXECUTABLE_NAME_UPPER}_FIND_REQUIRED TRUE) else() set(ROCM_${MODULE_NAME_UPPER}_${EXECUTABLE_NAME_UPPER}_FIND_REQUIRED FALSE) endif() if(${ROCM_FIND_QUIETLY}) set(ROCM_${MODULE_NAME_UPPER}_${EXECUTABLE_NAME_UPPER}_FIND_QUIETLY TRUE) else() set(ROCM_${MODULE_NAME_UPPER}_${EXECUTABLE_NAME_UPPER}_FIND_QUIETLY FALSE) endif() find_package_handle_standard_args(ROCM FAIL_MESSAGE "ROCM_${MODULE_NAME_UPPER}_${EXECUTABLE_NAME_UPPER} ${executable_name} executable could not be found. Please specify ROCM_ROOT or ${MODULE_NAME_UPPER}_ROOT." REQUIRED_VARS ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE) if(ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE) set(ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE ${ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE} PARENT_SCOPE) mark_as_advanced(ROCM_${EXECUTABLE_NAME_UPPER}_EXECUTABLE) endif() endfunction() # find compilers find_rocm_executable(hcc hcc) find_rocm_executable(hip hipcc) if(ROCM_HIPCC_EXECUTABLE AND ROCM_HCC_EXECUTABLE) set(ROCM_FOUND TRUE) else() set(ROCM_FOUND FALSE) return() endif() # find other executables and libraries find_rocm_executable(hcc hcc-config) find_rocm_executable(hip hipconfig) find_rocm_executable(hip hipify-perl) find_rcm_module(hcc LTO mcwamp mcwamp_cpu mcwamp_hsa hc_am) find_rcm_module(hip hip_hcc) find_rcm_module(rocm hsa-runtime64) # parse hip config execute_process(COMMAND ${ROCM_HIPCONFIG_EXECUTABLE} -P OUTPUT_VARIABLE ROCM_HIP_PLATFORM RESULT_VARIABLE RESULT_VALUE) if(NOT ${RESULT_VALUE} EQUAL 0) message(FATAL_ERROR "Error parsing platform identifier from hipconfig! Code: ${RESULT_VALUE}") endif() if(NOT ROCM_HIP_PLATFORM) message(FATAL_ERROR "Empty platform identifier from hipconfig!") endif() # set definitions if("${ROCM_HIP_PLATFORM}" STREQUAL "hcc") set(ROCM_DEFINITIONS -D__HIP_PLATFORM_HCC__) elseif("${ROCM_HIP_PLATFORM}" STREQUAL "nvcc") set(ROCM_DEFINITIONS -D__HIP_PLATFORM_NVCC__) else() message(FATAL_ERROR "Could not parse platform identifier from hipconfig! Value: ${ROCM_HIP_PLATFORM}") endif() # find libraries for each specified components foreach(module_name IN LISTS ROCM_FIND_COMPONENTS) # set required libaries for each module if("${module_name}" STREQUAL "hipblas") find_rcm_module(hipblas hipblas) elseif("${module_name}" STREQUAL "hipsparse") find_rcm_module(hipsparse hipsparse) elseif("${module_name}" STREQUAL "rocblas") find_rcm_module(rocblas rocblas) elseif("${module_name}" STREQUAL "rocsparse") find_rcm_module(rocsparse rocsparse) elseif("${module_name}" STREQUAL "rocfft") find_rcm_module(rocfft rocfft rocfft-device) else() message(FATAL_ERROR "Unrecognized component \"${module_name}\" in FindROCM module!") endif() endforeach() # Generates library compiled with hipcc # Usage: # rocm_hip_add_library( [STATIC | SHARED] [FLAGS] [OUTPUT_DIR] [INCLUDE_DIRS] ) macro(rocm_add_library) cmake_parse_arguments( ROCM_ARG "SHARED;STATIC" "OUTPUT_DIR" "FLAGS;INCLUDE_DIRS" ${ARGN} ) # allow either STATIC or SHARED if(ROCM_ARG_SHARED AND ROCM_ARG_STATIC) message(FATAL_ERROR "rocm_hip_add_library: library cannot by both static and shared!") endif() # default to SHARED if(NOT (ROCM_ARG_SHARED OR ROCM_ARG_STATIC)) set(ROCM_ARG_SHARED TRUE) endif() # default to current binary output directory if(NOT ROCM_ARG_OUTPUT_DIR) set(ROCM_ARG_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) endif() # parse positional arguments list(LENGTH ROCM_ARG_UNPARSED_ARGUMENTS NARGS) if(${NARGS} LESS 2) message(FATAL_ERROR "rocm_hip_add_library: Not enough arguments!") endif() list(GET ROCM_ARG_UNPARSED_ARGUMENTS 0 ROCM_ARG_NAME) list(REMOVE_AT ROCM_ARG_UNPARSED_ARGUMENTS 0) set(ROCM_ARG_SOURCES ${ROCM_ARG_UNPARSED_ARGUMENTS}) # generate include flags set(_ROCM_FULL_PATH_INCLUDE_FLAGS) foreach(_rocm_iternal_dir IN LISTS ROCM_ARG_INCLUDE_DIRS) if(NOT IS_ABSOLUTE ${_rocm_iternal_dir}) get_filename_component(_rocm_iternal_dir ${_rocm_iternal_dir} ABSOLUTE) endif() list(APPEND _ROCM_FULL_PATH_INCLUDE_FLAGS -I${_rocm_iternal_dir}) endforeach() # generate full path to source files unset(_ROCM_SOURCES) foreach(source IN LISTS ROCM_ARG_SOURCES) if(NOT IS_ABSOLUTE ${source}) get_filename_component(source ${source} ABSOLUTE) endif() set(_ROCM_SOURCES ${_ROCM_SOURCES} ${source}) endforeach() get_filename_component(ROCM_ARG_OUTPUT_DIR ${ROCM_ARG_OUTPUT_DIR} ABSOLUTE) # generate flags to use set(_ROCM_STD_FLAGS ${ROCM_ARG_FLAGS} ${ROCM_HIPCC_FLAGS}) if(_ROCM_STD_FLAGS) list(FILTER _ROCM_STD_FLAGS INCLUDE REGEX -std=) endif() set(_ROCM_FLAGS ${ROCM_ARG_FLAGS}) if(CMAKE_CXX_STANDARD AND NOT _ROCM_STD_FLAGS) list(APPEND _ROCM_FLAGS -std=c++${CMAKE_CXX_STANDARD}) endif() if(CMAKE_BUILD_TYPE) string(TOUPPER ${CMAKE_BUILD_TYPE} _ROCM_BUILD_TYPE_UPPER) list(APPEND _ROCM_FLAGS ${ROCM_HIPCC_FLAGS_${_ROCM_BUILD_TYPE_UPPER}}) endif() if(NOT ROCM_HIPCC_EXECUTABLE) message(FATAL_ERROR "HIPCC executable not found!") endif() if(ROCM_ARG_SHARED) set(_ROCM_FLAGS ${_ROCM_FLAGS} -fPIC) endif() # compile all files to .o set(_ROCM_OBJS) set(_ROCM_OBJ_TARGETS) foreach(_rocm_file IN LISTS _ROCM_SOURCES) # create output directory for .o file get_filename_component(_ROCM_CURRENT_DIR ${_rocm_file} DIRECTORY) file(RELATIVE_PATH _ROCM_CURRENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ${_ROCM_CURRENT_DIR}) set(_ROCM_OBJ_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${ROCM_ARG_NAME}.dir/${_ROCM_CURRENT_DIR}") file(MAKE_DIRECTORY ${_ROCM_OBJ_OUT_DIR}) # set .o name and path get_filename_component(_ROCM_FILE_NAME_ONLY ${_rocm_file} NAME) set(_ROCM_OBJ_FILE ${_ROCM_OBJ_OUT_DIR}/${_ROCM_FILE_NAME_ONLY}.o) list(APPEND _ROCM_OBJS ${_ROCM_OBJ_FILE}) list(APPEND _ROCM_OBJ_TARGETS HIP_TARGET_${_ROCM_FILE_NAME_ONLY}) # compile .o file add_custom_target(HIP_TARGET_${_ROCM_FILE_NAME_ONLY} COMMAND ${ROCM_HIPCC_EXECUTABLE} -c ${_rocm_file} -o ${_ROCM_OBJ_FILE} ${_ROCM_FLAGS} ${_ROCM_FULL_PATH_INCLUDE_FLAGS} WORKING_DIRECTORY ${_ROCM_OBJ_OUT_DIR} SOURCES ${_rocm_file}) endforeach() # compile shared library if(ROCM_ARG_SHARED) add_custom_target(HIP_TARGET_${ROCM_ARG_NAME} COMMAND ${ROCM_HIPCC_EXECUTABLE} ${_ROCM_OBJS} -fPIC --shared -o ${ROCM_ARG_OUTPUT_DIR}/lib${ROCM_ARG_NAME}.so ${_ROCM_FLAGS} ${_ROCM_FULL_PATH_INCLUDE_FLAGS} WORKING_DIRECTORY ${ROCM_ARG_OUTPUT_DIR}) add_library(${ROCM_ARG_NAME} INTERFACE) target_link_libraries(${ROCM_ARG_NAME} INTERFACE ${ROCM_ARG_OUTPUT_DIR}/lib${ROCM_ARG_NAME}.so) # add depencies add_dependencies(${ROCM_ARG_NAME} HIP_TARGET_${ROCM_ARG_NAME}) foreach(_rocm_target IN LISTS _ROCM_OBJ_TARGETS) add_dependencies(HIP_TARGET_${ROCM_ARG_NAME} ${_rocm_target}) endforeach() endif() # static library if(ROCM_ARG_STATIC) # create library from object files add_library(${ROCM_ARG_NAME} ${_ROCM_OBJS}) set_target_properties(${ROCM_ARG_NAME} PROPERTIES LINKER_LANGUAGE CXX) set_source_files_properties( ${_ROCM_OBJS} PROPERTIES EXTERNAL_OBJECT true GENERATED true ) # add dependencies foreach(_rocm_target IN LISTS _ROCM_OBJ_TARGETS) add_dependencies(${ROCM_ARG_NAME} ${_rocm_target}) endforeach() endif() endmacro() # Generates compiled objects # Usage: # rocm_hip_add_library( [FLAGS] [INCLUDE_DIRS] ) macro(rocm_generate_objects) cmake_parse_arguments( ROCM_ARG "" # options "" # single value args "FLAGS;INCLUDE_DIRS" # multi value args ${ARGN} ) set(ROCM_ARG_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) # parse positional arguments list(LENGTH ROCM_ARG_UNPARSED_ARGUMENTS NARGS) if(${NARGS} LESS 3) message(FATAL_ERROR "rocm_hip_add_library: Not enough arguments!") endif() list(GET ROCM_ARG_UNPARSED_ARGUMENTS 0 ROCM_ARG_OBJECT_VAR) list(REMOVE_AT ROCM_ARG_UNPARSED_ARGUMENTS 0) list(GET ROCM_ARG_UNPARSED_ARGUMENTS 0 ROCM_ARG_TARGETS_VAR) list(REMOVE_AT ROCM_ARG_UNPARSED_ARGUMENTS 0) set(ROCM_ARG_SOURCES ${ROCM_ARG_UNPARSED_ARGUMENTS}) # generate include flags set(_ROCM_FULL_PATH_INCLUDE_FLAGS) foreach(_rocm_iternal_dir IN LISTS ROCM_ARG_INCLUDE_DIRS) if(NOT IS_ABSOLUTE ${_rocm_iternal_dir}) get_filename_component(_rocm_iternal_dir ${_rocm_iternal_dir} ABSOLUTE) endif() list(APPEND _ROCM_FULL_PATH_INCLUDE_FLAGS -I${_rocm_iternal_dir}) endforeach() # generate full path to source files unset(_ROCM_SOURCES) foreach(source IN LISTS ROCM_ARG_SOURCES) if(NOT IS_ABSOLUTE ${source}) get_filename_component(source ${source} ABSOLUTE) endif() set(_ROCM_SOURCES ${_ROCM_SOURCES} ${source}) endforeach() get_filename_component(ROCM_ARG_OUTPUT_DIR ${ROCM_ARG_OUTPUT_DIR} ABSOLUTE) # generate flags to use set(_ROCM_STD_FLAGS ${ROCM_ARG_FLAGS} ${ROCM_HIPCC_FLAGS}) if(_ROCM_STD_FLAGS) list(FILTER _ROCM_STD_FLAGS INCLUDE REGEX -std=) endif() set(_ROCM_FLAGS ${ROCM_ARG_FLAGS}) if(CMAKE_CXX_STANDARD AND NOT _ROCM_STD_FLAGS) list(APPEND _ROCM_FLAGS -std=c++${CMAKE_CXX_STANDARD}) endif() if(CMAKE_BUILD_TYPE) string(TOUPPER ${CMAKE_BUILD_TYPE} _ROCM_BUILD_TYPE_UPPER) list(APPEND _ROCM_FLAGS ${ROCM_HIPCC_FLAGS_${_ROCM_BUILD_TYPE_UPPER}}) endif() if(NOT ROCM_HIPCC_EXECUTABLE) message(FATAL_ERROR "HIPCC executable not found!") endif() # compile all files to .o set(_ROCM_OBJS) set(_ROCM_OBJ_TARGETS) foreach(_rocm_file IN LISTS _ROCM_SOURCES) get_filename_component(_ROCM_FILE_NAME_ONLY ${_rocm_file} NAME) set(_ROCM_TARGET_NAME hip_target_${_ROCM_FILE_NAME_ONLY}) # create output directory for .o file get_filename_component(_ROCM_CURRENT_DIR ${_rocm_file} DIRECTORY) file(RELATIVE_PATH _ROCM_CURRENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ${_ROCM_CURRENT_DIR}) set(_ROCM_OBJ_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_ROCM_TARGET_NAME}.dir") file(MAKE_DIRECTORY ${_ROCM_OBJ_OUT_DIR}) # set .o name and path set(_ROCM_OBJ_FILE ${_ROCM_OBJ_OUT_DIR}/${_ROCM_FILE_NAME_ONLY}.o) list(APPEND _ROCM_OBJS ${_ROCM_OBJ_FILE}) list(APPEND _ROCM_OBJ_TARGETS ${_ROCM_TARGET_NAME}) # compile .o file add_custom_target(${_ROCM_TARGET_NAME} ALL COMMAND ${ROCM_HIPCC_EXECUTABLE} -c ${_rocm_file} -o ${_ROCM_OBJ_FILE} ${_ROCM_FLAGS} ${_ROCM_FULL_PATH_INCLUDE_FLAGS} WORKING_DIRECTORY ${_ROCM_OBJ_OUT_DIR} SOURCES ${_rocm_file}) endforeach() set_source_files_properties( ${_ROCM_OBJS} PROPERTIES EXTERNAL_OBJECT true GENERATED true ) # set input variable to object files set(${ROCM_ARG_OBJECT_VAR} ${_ROCM_OBJS}) # set targets for dependency management set(${ROCM_ARG_TARGETS_VAR} ${_ROCM_OBJ_TARGETS}) endmacro() SpFFT-0.9.10/docs/000077500000000000000000000000001357247005600134645ustar00rootroot00000000000000SpFFT-0.9.10/docs/Doxyfile000066400000000000000000000007541357247005600152000ustar00rootroot00000000000000PROJECT_NAME = "SpFFT" XML_OUTPUT = xml INPUT = ../include INCLUDE_PATH = ../include ../src GENERATE_LATEX = NO GENERATE_MAN = NO GENERATE_RTF = NO CASE_SENSE_NAMES = NO GENERATE_HTML = NO GENERATE_XML = YES RECURSIVE = YES QUIET = YES JAVADOC_AUTOBRIEF = YES WARN_IF_UNDOCUMENTED = NO MACRO_EXPANSION = YES PREDEFINED = "SPFFT_MPI" "SPFFT_SINGLE_PRECISION" "SPFFT_EXPORT" EXTRACT_PRIVATE = NO EXTRACT_ALL = YES SpFFT-0.9.10/docs/Makefile000066400000000000000000000011111357247005600151160ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) SpFFT-0.9.10/docs/images/000077500000000000000000000000001357247005600147315ustar00rootroot00000000000000SpFFT-0.9.10/docs/images/sparse_to_dense.png000066400000000000000000006733731357247005600206370ustar00rootroot00000000000000‰PNG  IHDRGÃÆ?tç IDATxœì½iו&vÎÍ|KU¡ªX(öX\RâÒØÝl¥&ÕÓmК–4#ù'<žð8¶Ãa›ø1?<Ï8¢Ó­h·ž^"O[t4G ¢$R$(‘D$‚ ö­°Ôþ–¼Ç‘¯^fž»ä{¯°UÀùB"^eÞ¼yïÍíœï~ç\@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @0ï€rÉ·9ðá‡.oØ°aéäääâ©©©ËõzýÔ~ô£‰Û}`@ ×"Âõì³}½Ai«¦hs]×?ª×ßýÿã‘!æ&„Ü–ˆ?XÏ>ûl¹T*­( ÛµÖwOMMõV«Õœ¯V«'”R#âѱ±±ñ}ûöMË"@ h…ØÆÜóÄ?훪ݧ•þø ¢"¢i:4Y©þZ)|=BúYý2^üç/ÿÕ¨ ¨@07 äˆà¶ÃSO=UZ¹rå†z½þm&¢žx *• T«UÐZ7†#­õ8"ž€CµZíÈG}têÈ‘#¹k@ ÿ÷ïý·=zñ…ˆêÛúwÕ¢d7Ŷeì|‘¨Ñy<H{±¦^{é½£¿Ù3¼o\T øü äˆàvzî¹çzëõúZ"ú¬ÓZwó¾OOOC­VKÉD4…ˆ§ƒ ˆ‰’Cpîïþîïbò„ä.@ ¸ý°{çÎpÙÊMCeUÿ}€#Ä ŠgÚÎ¥¿1',&K" « ð½¢4¼Ráð§kמ߽{·ß(7BŽnilß¾½°iÓ¦…Åbq³Rê!Xªµ.ùú“#±z¤CT´Öç …ÂI":BDŸž={vìÉ'Ÿ¬<ÿüó$„‰@ Á­‹óð®®þõ½«C 'éÛ°jd²œ¬ôoN–x+K€`áo@áϦuíM:]>÷7õC{÷îÄÆn„Ürˆc=wìØ®Y³fy__ßýDt".Š··ê«VÓ)QÑ("^@ÄkµÚÇ—.]:µ|ùòÉ_|±Q™|È@ æ79ëy¤üû+ïº[uEÿY¤é¥6ĺNx4HØ «Éï~ƒ±­E¢Qч Ÿ×"zí7gN¼÷ú•+—ž~úéH&ä‚ !G·BäêÕ«]<òÈR¸G)µj­U'}ŒÉ‘X=rPAÄsDt´Z­~\.—/þæ7¿«V«Õ‡~¸ºgÏžHî:@ ‚¹Ý»w«}/¼Püú ôvõÞUPøuü­Ę2„³>'+&M´ÎH’øoŸŠ“£›ÿÄebœ«$ÎIr4ü)z}J×>þ·¯½vA+5Yß¿Mn)àÚ!äˆ`^#þ`íÙ³§{ÅŠCƒƒƒ+Êåòz­õ¥T_§¤H‚HŽ€R*V Äl~%V•(¥NÑÅJ¥rêÀ—ŠÅâØôôô¸%@ s»ví N½óÎà?Ýò•õÑÂú=_F€/Ç!ÚÆTHBn$JþÛ†¯lƒÁ,ÌÆ8676@5¡1<èã:ÕÞû?ùÖ±êçB¢ —&…,f!Gó¸}ûö0 Ã…ƒƒƒ«ïTJ­FÄ%DÔu­ýé”i-÷sÄeY2©µ¾¢”QJ]¸|ùòùóçÏŸ ÃpdãÆcB”@ |>ˆ'Ýö¿ôR¹°úë+ï~ Z@¨·#àèçùBlÄ$FlI²éX“í½ á’!^5J–À$ÝK€ .Òi8 ¤Oúé±>* ŸTÇÇÏï®Êm%´†#‚yƒøƒõÃþ°T*•V¬]»v ¬j"}D^o?n”r$!O’$‡H‰¿fqR­ø„ñŠ7/^¼xfjjê8œ]´hÑå—_~Y– @ ¸ÉˆU"+.Ôz7,,ÞÓÕ]|2Äp;~.­K³Qˆ˜ÈŒ”ô€æ5ä*OÌøj&"Œ"i–KÃnRr$;¬Q5b€Fàh:Z©U|pþÂþ*DLLOŸø‹Ÿÿ|Lî)À…#‚98ŸÈc=¶`åÊ•K¥Ò}M•È(Ì6t¦fKŽ´StrLB Åa¤P×ZÇËÀ™8©+œˆ&ccc"@ ¸qˆmÌÿýÉ?ê^Pz|AwéQáƒ8+‘ ˜9‘EpXh®Òk„½ð\"Æo`™WY] ‚$ÝÇ88-’P%”*N¬£fVlj`G è,QtàÐ… ï†îtu]ú?öì™’ÛJ rD0‡+EŽ?ÞS©T¾†áCˆ¸ÊíV¹V´[­¦2ÄG~ØÇ´*“„ßĪ’ b²$V\€sÕjõHÇN:uAˆ@ ‚kÃî;Ãî¡¡EÝPxbA©ð-Tø¢ê£æ¤ZÄ…½ XˆLR†Mxeå)S‰WwXä p5ŠEœØçãudd2Ö$Û–IÕ)D5h*— èhøp¢Z{]!¾ñƒCÃG^Ú¿Rn+Áí !GsÛ·o/lÞ¼ym-‹_TJ-ÓZ—nv;;]Ê×Cf8ûìšvÇ€'ü†ÕEñ¿AÄÿNQ¬(9DD.¼ð 7&‹¬@ Á-ŒÿkçwÊÑòê’‹O!Ò?Q¨¶@¥Lu±y g»•4Õ “ëX–sœzãM*%\œ\¬IÝŒ”1ê1¢pX²W°öSBÈ4ˆÒ€¤téK á=Mz/èúš¬T>~îÅGe©`Áí!Gs;wî ׯ_¿"Š¢íJ©{¢(ü,ÛÖ)9³HÊÚ‰ ÅWÖG¦Xÿ6’»ÆŠ"úOLLL|ò£ýhb–Ý@ ¸¥ñ'OýW¥Þ…c÷ ÿý"®Õ3Ü€ÓíV¹Dìü É`$Mõ®JFØŒ™2Äd`ÐH´ é±F:l–3ÂhÐLâš’%ÄR–`Zon/‰""¸‚ˆÒIÓ~U›~ãÿæo.ÊS"¸Õ!äˆàsÇSO=UZ¹råšZ­¶ƒˆb¿f•ìêÆÀGŽøˆ è€È€¤GŸR„ïóÕÅÁÛÓÌY¯€s”ˆ>ššš:vüøñ«Ã’™\ ÁmŠÿâÑG{[·éAEúð÷aI2-“«²p˜v¶(Ïûܹâ$ ‘EH —'1Ó¬f‰WyN¦Záíà¡@ÀÂkMJ£M*U«˜}aýÈvÖ èüœH½ Þxuäô©ï¿ô’„ßn99"ø\ÐX*mÿþrÿªB¡ð%­õæ8É*Ï'òy‘#>åG^°Hvá2à!=|j¾ÏW·ç<¤”ºJDljè°R*ÎU2Õ­[·ÒóÏ?O"‘@p+"^yæ&õÕû+Ûëß…¿ÛH²Úð÷1ÍÁá£1Í£ÄmxHÿ6äVØ 0¥Hü¯nÖo[µFÒV¦!FT$¹K8Ab¦A¦ IêD‡øpÕ"YÞoƒ4jö“2¢EéS¸_E¾ZVåOþåû?›X·n~ñŵؘ‚ù !GŸbâcÛ¶m…žõëׯ‹Ó‹ â†8#ø\¸ œÉ#EÀ£êÈC«Ðš>55uúwÞ[¼xqu×®]u!K@ ÌgÄ“nßÿþ÷ËÿzÛ“ƒÓKèQúˆÁN"êÏ“v>éæ[fדIÀ¥‚E°¸I\yJI¾Ï!Xx»šû%òW§±—öeÄ ¯)kcúƒ&Ù“äWñ5Pè Dð"¾E„oÑôÿæÿ}å,ôÂä¦M›ª{÷îÄÆÌ'9"¸©ˆ ‘Ç<8yòd÷C=´¨P(Ü [ãåx?‹$«³A§ää+6œ²­"íh£2ÉC^Ì\ˆˆ."âE":EÑ©±±±‘C‡Æ±}ûöÕoÈ` @ ÜDÄ„È›ù—…cµÚÂÿ~ûÎuº¿FˆÛ¨7=s–¥ÔQ‰ˆ•ÿcF-¢sVŽÉ–äõ®>Ó¡:%SŒ¤µš [íœ"ù³wß=W.—'>\¢D0×!äˆà¦ &EvìØÑÕßß?´xñâÅbqRêN"Z!|a3í“#ÓÓÙÂ/yÄD«Ð»ŒïØNÔ"vY»žfR×8ë¥XQBD'=z¶Z­^ÕZÝÿýõ={öD7i¸@ f˜ùÉ_ü‡þon¹cÏ"u¯xD𬂲MT³´­¹¯¥ºƒogd0‚À$E2²Ä^ÌÆQs¤ä†Ê”'VVß’ÂÀˆ“<;Ú»p‹,¬ö =I+Z/fÿÌÜ(¤u¨.#ècj?éê;?<ðáGÇ.Ÿ …óCÕ=’O0!äˆà†"&E{ì±åryÕ’%K6†a¸–"¢7tf.$WœÄÄÜg„ÝX¡5ærW«Õ>&¢O …Âù—dÝ{@ ׈?yꩾJ¹¼cq±ÿ H¿‹ V!B°‰7wZ;?G>`ª9Ø9E8;úŒ(ádˆ¡ ±É_ßj3ÙN–ZÄ—Ø5;T5s–˜Góo2åFºõÇHË:×6êMÇÖÈXä¿>`’H9k 7ÏG›±DÓ€ð |pqlò¤¢7à øèŸýõ__Î0à†@ÈALj×€¥Ri}†_ÒZ¯"¢nÕôÂ}„F'$ÇœÈ;¢ A5*•*Sãu]«˜ê‘V‰Yy¹<´R˜øêmGˆ\+ir“@ÔPI6r–Ä9JŽ)¥W«Õãc~# žÛþ\á‹ëC‘ª}¹¨Ô?Äí p€‚,CHæp{Ä6¦bËårä…Þª°I tVˆI)À}6'Ð ¾Õk€§™åÄàKËB\Œ¤°ÎŠ4id-5 V{…ÉÉŽ «:;2;SaÒl7"!麈8\©ÕßP@¯ßþéñCç_Ø·OT%‚ !Gm'Àúå/yÇÒ¥K·i­DÄUˆ$Çå½´]Ö¼õË=OîxSÉU¨0دKK7é₵Q•j'ê“—ÏêÉÓ¡viÊGl\KK»|$­Êv‚k ŹYˆU%A\&¢ãða¼lp†#²ú@ ‚ñÄÛWk=« žÔßT*x`&lƆ¹ÒL‚„¼ð.?ë³-óX +!)'^xÊ^ ¸(‚/œœŸåæ 3ǧqWÚñ-ˆ×g«9ÀOŒäØÕ<5Š±Ù>6N™ÃÏÝ,’Ž1ÆIXZÄ'àäxqWjv7ÖöT‰ð¸R°"ú1Ößÿ«ƒ¿:*!Þ‚!G¹ˆWžùæ7¿yG¡P¸€•D&å}*°È_¹6çülT$ªPa _——¬£bßFRÅ;âÙ öÒ@ׯè驳ÑÔȱúÄ©sX½\ˆÎ};%ˆ½c–«ÈÌ sDIb V• 6oÅËÁÇÓÓÓ—%üF ‚ÛñÄÛºáá•õR×3 Ô7á~(%ƒÑI2QþÛIªšÃ*p%„“4ÔýÈqòäGŠ¤gp'}$ˆ¹,p¢ðV¾©žá¡(VxX¶8LGNH‘•>4»¯XÒ––Äl?Z]NÈ=‹«Ì1–.¶Æ]Šº&:«ß®ëè•B¨^¯…á§ü§z%w/–@ÐBŽ Ä„È#¬å’í0;/Š±Ì±—»²e*~ŸÂKƒžéDˆp•4~oE¤ß  ýâpmôÌó/¾X²DÐ „4‘Ǽ‘OdÍš5+à¥Ô´Ö óFg¶«Ð´;n¶!8³‚ På.*.Y•®B×ZÀ°¬Iìƒg~˜­vÖ)ªBmúlT;[;u^W.N¨ëºZ‹‰ôOu´"E| `;]Nx>"yˆÉ­õˆRê$}P¯×¿÷Þ{—Ö®][ß¹s§~þùçI>f@ ÌOÄ6æÆ‹ÿóêÕ‹hñªû0À¯Ò“€¸Útýp&ܼ =2ù‚Ax¤›ó’£¢sœÁC0gÒzÐ^ä M m¤}lccæü=³®oy];¿0Â#?fÆ éÁ8¡«AŽpRÄJÔê)‹Ô`j[D’.÷Ëþæ׈¯ÊcÛèˆx8 †@Ÿà¯1¢ÿ„€o}ÿýýÇ]½:½råÊhïÞ½‘ؘBŽÜƈ%/¼ðBqóæÍK—.]“"ˆ¸†ˆzìQq䋳$.ìA¾¬ðÆ"ب¤è «‡J‹—éÒ¢ ƒ º³[öRçJ'žÕÛžø$ÐÑ$èÚˆ®N^ˆ&΋*WF1Ÿ z­ T‹øc6 \oGÄD Åá7§µÖŸDQtº^¯_=pàÀ¨ÖzúᇮŠºD ‚¹X%òÞÞ½ÝßÝþ•Õˆú#Ð_„/" 5ñ–—_‚ ’Ú䦋s]h+'ˆ½Ä®‡"Hÿ6›dΚ9jt4†77gŠÑpÊ+Ñ«·©ãHµ»ÀÉ;çˆ'ÆT¯ Án$õ$ÿÚ*ƒ ±Õ Fr”3¾ñð«F0½< —ãDä°Nö;B4€÷5À/HÃû¥ÚÄ©ÿeïÞsårù* LJÎ9r{"&E^zé¥Þ+V,íïï_A¬YˆE߀äÅ0ÚÛò0ÛКëALŠ*H-è×¥E˨8°–ÂÒ@Uó;É^Ôf?‰¤‘A;Cö‘Œ Å6Q IEÕÉóõñ³'ôèÑcº6Y½UU7ñŘRJ]— VJ›šš:3<<<¢µ¾:444þöÛo×…ñ`î`çÎáʱ±¡'ÖnÝuãoOÄ“oDÐÓn,/ÏH–ÃCœ0²‚<)xògø—ÑE¦šð™Ùòºã“Ûu%ùº ’%n}a"ÜÆÌŸJ„«82×HP›5ÝC¢˜‚¼ðœ´,·éízXòT4n>Zÿ¹ºÛ$q”uuÝßø·?å“€èTmñâ‹O>ùde÷îÝ欦඀#·¶oß^(•J‹/^Ûßß¿îDÄEZëÆÊ3>EGÞ,½Ï+go q½@ T=¸c u/] …þåE@ÊÎÔxI¢ùÂÎbX¼¥d÷ 7’Õ¥ (ªŒŸ®_xÿ§zòÌU!@®MUÉDœØ5&K´Ö§GFFÎNLL\èïï¿´O–ràsÃwvî,O]ž\ýÔºm_Ò]ÑÎ@á#t'óB#|È\}KÐ8΢8¬PŒÄ9·WŒ™Ùj«=¨…„ÌÉgçDJ閈߭Új¿óžÏèoÍÆ1þ©+㧣 ~J•sWnzÇoQøTJÉ#b,}Gız½~öܹs'b¤««ëÂ… &‡‡‡«·ûø @p3ñÜöç Ñ¢O{ •{Y±æQ]Ð!†Û€ôb(ú&Ûøl>0„Ÿ´\z6Ñ“1òQ¤uAö‹Dn[xr“ØäˆaääÄ0“¸r½Š2°UøøLNö‹Ào»óð"ß9 JRKº}R+§ˆg¬[/êãשØD‘{®t„[’jî9á>¶ß ‘d \I‘¢)D!€³T§ƒož8öN5Òï!ÔTz{/}_V[¼¥!äÈ-Š˜©×냅Bacww÷æ(ŠV"â"*©)C«| yð­ê½åò@v_‰ýƒ‹1ù*(€*vA¡Ô AÑ$J™²[@–0ùˆ6Hr^ÿ¤+§£ ïï¦Î\±W« ©ñ"/AoÞoV6¯©À$"^žššº0==}*Š¢3ÅbñLÿ´ä*àÆà϶?W8³äÂkô…Jц÷!Ñ" è#„€Ÿ$qtuNBÑ<¤„Aš,ÕMÄŠ†ƒžyêf^Î¥02‚ÕŸæ#¡ì;Gx'[+a ¢Ç"„øv“õŒ„«¬!F»ÐBYŸ ‚(w¼íxš¼€!×FÎì2;Ü%#òŽ‘Aœ¥÷šÄK2u–\k>’-gl}*Þ'¾¯y¦ŽÀe@:®5>rþÜ{!žÒãgÎ\Ù½o_=g`óBŽÜbˆc=‡††–”Ëå{q+ @œK$hÕÓ<'ìoaÒ.´¦]0Ɉk‘ê¾-~RC¸¸,‚°ªP‚0&J %ÂBœåj¦]Þ¯­õÑ´”!”°Í@ÆG )UÆNé Ãû刽Ôï톼8b~ÿt¢êÉÛÆf¤Hk)¥êZë*"^Œ¢èššš:þðÃ_‘R@ f8tæKK–l.]ÿ|lŒs‰QÁˆ±ã2ZÀvîmÂ$!ü:ß 2YÉ”`çp9€ŒBàŽ9ÿ;-žC†Ø¹-€¹é¹aÉ/¸ƒn„‡Ø šâ" ‹±ò¯¤ý°V‹1ù‚ŒP0§ ¹rÆ‹r±">e ZI]\Ÿ 7ÿŠ GЂFã _‚•å×Xâ\Þ—,Ç‹olE# ¨¡Â*i=¡ лµZ´Wìÿ‹GNì¢dÞCÈ‘[ñ2i_ýêW‹KÃ0|¶!âB" fÞ?íI‰NÐŽ@iU»œVå;ÁÌ -ì¹ ý^Dá Ÿ£  þ_üoP,CXê°“%ŬOà~Ú;uQúÑ4ÎÁ>lñ*6T?]~•*g r$%þûǧòevo)›ôç̉ ‘8™ë§ÞºÓZ IDATQ ×jµO`ôå—_®|ý`>"¶1÷<ûlÏhW×]†ÿQ}‰Ö!bAåÛ˜N¸‚ ¿ªÃL:š©xÝLía«L8‘Úg¾•bLÒ Äáö„IäMþprBtx^;„xÒQGCa©8Ìü&Ùà’F.)VY䜋y¬3&éh县¶Þû†5$íw’œÖÃG‘c+‹ÒóYÍ3ó¬ Zã„:1ÒF¡9&Ò3ÕAÃ'ô›àeUߊW\üÇÿþßO¸Ìu92ϱk×®b__ßêZ­¶·Ñ­féÛÍà·:f6ê¨G®G9’#£ÑÐ5®tÏékª°A”Ê=Ī’°ÀBiš§¦¤sæwòÔ›YpúNQeô„ùà5šrsŽÜN„ˆoì[Ý› òÉæ36Z$1tœØ5Š¢ A|HDŸT«ÕS/I ©@ )^zúéîs½ w„Š¾€OÀJ#\¡ ç;K¶Cêè°i #5xˆ¯ËvÇÁQudg5m_Þ‹V+¬pÕ doó© ÇÆÍB|¢Æ¿­ 1ȦÔFï˜ãí\ SGÍÇÄ 3JûÂý~°ÏÃô«Qø1‰hÛG’8Ä “ÅäÚ†Â%Ëfb\C¶2Pš{%ÛÞÏ8_2®iu 8«IT¯R¿¿ò›ö×}ÙtÁ\„#óÏ=÷\wµZ]MDÑfèkÕ“'ë£`UÇNè‘á9ÒŠ ¹•#$íBp8ZÝÃ^CÄÚÖLîz6Š¢Oà`†'âd¯’§D ·#þêÿ˪šÚA@ÿXü. ,ÏwÑ”Pc6]¥! †_ÙLüi:þ69>–ƒÕ‘•ËB"È%K’pŽ–JTLÏʼnÛfŒ÷j–7Ã5-…'x¬Ri¢TƒÐ°È•ÄÂ̳eì±JÆÐKeKß&ãì0;¤(í¿ux8Ô0ˆ°„ü°Hƒz±®ÜzÈ3>è*qȸœf’UCóœ±g¬ŒKÐÙ¢‚c…äd…h>Ô¿Ä~X£Úþ¿}óÍ‹{dá€9 !Gæâ¥x×­[×ÛÝݽ…ˆî'¢UˆØ=›´"8Ú•··ƒõ!É«³Ýl'褼¦°{¬¾è·k®õío§t1>†*€îÁU€qn’¥Œ}LF{+¢é±“õKïÒÔÙ+vÍ힌5OM’Gz´:ÖÞ75“}®f¹øcuIk}, ÃÃðÉÑ£G¯¬[·N¿ø⋱âdv²+@ ævïüNyÍZX ü úCzQ ßIÈÉŸÁCT¸Ò€—­0‘Œ1ÃNb¨ä[ŸÜ Ȫ/ñŠ3µ€I˜øH–jæD»*› WíÑÁªŽh©.ìÐ狱°•"öÒ1üLÆxØÄŠÇNÏ[‘ǃì 3ô%íŸú’œ‹Æî)s àŒlÈøûÆc ÆŠ˜Ç#«R2Ä^ÂØV(Š+¼(S#)V¯Y–]°t{#‡`>‚7¢}T«½ñ_ÿä''—,YRú駣矞ÄÎüü!äÈGë¹cÇŽpݺu‹{zzî€ûâ¥x¯õÚµ";:™Ï•¶§éTAÒ©j$¯þx)ß«ÑàïÖ¡xgër8̈ r$¹á/Uþ²KÛ ÖKŸ0VŽœÒ#Ãûì°šÛ™-ZÏú€±½"ÊF«û ¾FD'wW:«”ú˜ˆNT*•‘wÞygtÆ µ½{÷FòÁ|Elc~÷ñÇK®Þ•ð…ðGˆp/[ ĵ¼!Ͳyù1l{’™‘c¦hkc*0û¬úàƒºî½÷ÞÅÅbq«Ö:&F·[u¦:Q‹p´ú ùÊu2Û?ÅH+ØU¥‚•ÚÐo× °Ñ×Þ<çÙnw¼-þ¸t ®l#\“—Ŧo7öHk ]8]|¯/çÈ톼{£©áC«pšV×:³%Ìâå‚ãÕo´Ög´ÖÇÉ]ßÿýQ¥Ôô·¾õ­š¬‚#‚¹ŽÝ»w«={ö„CµZßwî»o3»¿Š ŸT›ãUgL…çûÈÃ:ød9ÙŽ¹É!؃çûm”±œ]s_r´¸œâö Ë bœÐZ¥†«HLÕ˜Ž2WWXA(hÙ˜¶rÆ×V;¤µãŽ¶kG3žÉ 1ʶRºÍ^‰†o7ÊXäZªžAÏoOn—L“Ý›*í§Ù!7ñjv!ì{ #JŒQ5¯ë¬öª7¦rĵÑ"W²ß0®>&ÃÂÏêT{땃ϼyèÐèÐÖ­Ó² Îg!GæâÖßþíßv-^¼x0^y¦T*mDÄõJ©" gÛÒ¼™v×1íDM’”kþЩңSÇtvá7*¸R|¼FÅ/ä|IJŠ½¬q²šäHX*›ŸdF» r&ŒÉ‘ÚÈŸâm@Ž´ºO| ¤¼ÐªkQx\OY¾Ð§ø_hî¦ü‰¢¨®”ºJDñäÔÔÔÉ>øàb©TŸžžžxðÁk’³D s»ví .½ÿþß^ÿŵÑ@´5 |@= WQ f3ÙÀrS8¤f¶•9n;¸`9Ë>•H¢pà½Ykö7s–YA’ÔîïŸ%6È›äñ™i³ÆñåÒZ&I¦â$gõ^E‚c%¹Âëã×ÁHþaqJ i¦öpÕæ˜ú Þ^W-â*Êù~¾%k˜ßÆå“®é"< ‡‘è×u¥ßý‹×_ÿ˜êõ e€‹£Ë–M Yró äÈÀÎ;ÃÉÉÉþE‹­Z´hÑXKDKqÁÍj]§á Z‘­â8yûܳÝo[5Z_¸³ª‹wMŽ´?‘I#ÇäÈÂ3äãEr›b}{¨2qº>r৷²r¤“ð«NÔ"Ðâ:Ï:ê:G†€gÿ˵֓J©Ëõzýb¡P87==}éã?>W*•®ÀÕýû÷×nZÃ@ ÈÁwvî,L,zýŠûuOïCˆøEDu ¡y«®@6ñ㳩L·ÕˆK 0Û³ÅE3çúÙ¹:T~¤g·'ôÁr¸[؊Ʋ±–º#mcÂÑbáÖ$ïÒ³Ù p›= sÖ£Â0ˆ&C)BIx¸ÕŽ”¥°"TlÕˆy}ÛÁß6ÈŽ·}G‰cÝ(Î¥´¯ŸUИÝtø'÷2Ý/w\;ßYI‡wöÉêLZ_Â}…€Žƒ†£ôAœ®ñÃáPëS]½½§^Ø·oºí` :ƬÕ‚ƒX%²oß¾x9”¥+V¬Ø¤”ZCD˱—ˆJ7Ó ‹¬ÈS—ðºB£A’/o¼6‡×{Eqx¢pô–Çy>RŒ™FkÝ|¶ÜX®ô”ò¿’ó6éÁ¯k'áZ„jµR=ÝhØ9`òrÂpR„—‹Ãà±'Š¢¥Ô ­u­P(LoÙ²eLk=V­VÏ–ËåS…BáäØØØ¥·ß~{JâH@p³«DÖ]¾¼`²^ßòÀÒå¿M+ ;ក` ,ϤŸu‡”™2›Ç‘Ì6]\#·|Žiâ§ÆóЭ%e™%Ù Í0-›©·û”Õ›æ°Hì5VÖlŸ‰¤\>ÁDNó¹¢,W9³3™ú! ±±Ú`äèÈB12›œ³œkÓ¼p (®IšcÚ[>Õ5±]ž¼ɽÃÇÇÀC!·S¢Ê¤Òqô%Ʀ´yYÃ3EKF‘¤W =ÖU~ðöq˜ 4J¹Ê’f›ÇäLBgÏîb³±§S‚AD¼/€` ººÇ¾qÿŽ³ô¹j—ºß !xW‡ú“òƒ^•0ïëƒ(G>cÄŽÔã?Þ344tg±X¼[)µ¡©)^o>‘<ØŽf^HC§ÇûêÉËAâÃlU#íÂn²óa8Vü­ ”·úÂ8ÚŽ“õ’²sŽøXöä=i~Øâœ#㧣‘’s¤ |ªȹö×£6JˆVu´RŽøʵK¬— ‚ V•DJ©jEZëó“““Çñ“b±xú?øÁD\tVƒ&±ù¯|°kpñÃ=¥â3 Â¯4òÕ!tÇùD<έýÍËûFGŽÀ—ÒÍ[©%ÛÕåžãÉÍ`œÖXÂ×®Ú _ÈËMÀm¹™å‡¹*Ä«a¡.F˜‡•ƒ…“¦ÙèU¸¤‰CDñ¾¹—Æ“bíçy?ì¡°ò˜GYa-©šÚ\)&#ÆLÂÅ«†qTfîκp ̓=*“÷ñ­JcôÜÜæYŠ×&EÒN`x&^${Œ’ñ÷2iS²}MÚ1Ûl—W¨ @tQ:yyäÍÁÕÚôûK¥‹ÏJˆ÷¬!äÈgˆoûÛ=ÕjuƒRêÁB¡°zæâ5hÁËuJÓ®œNÂtìóàLÕÂhmá£(okéT{7ñö&mîj¬VS2>¶üÒøðfuŠ*£§ôŃû¨rk‘#¾p¬Ùc­î)Y7Û{Ç>'?âýA8äG;RÄÞÇ ;ìÆ—»µR*Š¢¨秦¦Î)¥ŽÀ©‘‘‘K¿üå/§:ê @ Ù7¿ÿÌ3ƒ•R÷—{K¥o"Âc8¨ ‚äSèLFxìW‘ˬTáÉéÀ_‚©uÈ™‘÷$µ]Å´-ÊqHSU†—ŒACÒ‰Ây7g0§ÝÃÆ"Ž8Ib‡P$•˜$«Õ±1ó“xò±TÊTи;?†….ãpž±¶Oe¨ZÀ­‡»qœÙQCé‚™:'SV‡¡D±î/—òB‹¸`Í0”:îf9G<¤“³4µE‘åð3˜ª|,ºÌ&š,þÈz®#B¨RDã€t¸V¯¼2=õš‚ð½‰"û/¼ ¶@È‘›µk×®Þb±‡Îl€;±{¾4Þ÷‚m¥D±I‰¼ò~֗âs§8(Žé…LGÅmö¹xÝ6/Ós(/\…R·™a;ã’ÍpöÒ¦8§2~²zqøU¼È‘Ù„¸ø®½<×¾SìZ’¼|!³Ià#CÀ"K8’2Í\%ÔTŽLÅKk­‹ÅjµÚ™x›$uAvïÜ.Y½z N×/……_Tˆ ©…}oç±;¿ G“[P¶…àžÜtþU„-&HKfRöÍ÷[o-mÌœ¤¼JTÖRºœüð)$€Cjþ%ý³Tž10#öˆeä@Z/³]íee}ú73 àjŸf/m"º€Ž½ `Øƾ„©¦"ˆd´Œ36 `(’¨ÙS‹´ðÙ”–Ãæ1ÍêuÃh‘’3œ2r· C¸˜'e‡2YBÊ.®19k´=oÙèf¡x2n&+-DDú*hø Bø…ü Néá?9ñá%Iêê‡#7 q’Õ;ï¼sYE±£ÿõ|jÑÑI™V„ÇlÊòót¢ 4®žÖ¥­ØÚÓöÇD/T„®…Ë!(uÒ8WgÎV$ßÕ¨:z¢>rðµÛmµ{;G'÷x 3°>t)í—C‘§±Ëûzã#R¬1©ÑE"ú?ÿôïÿþïÇ$O‰@ büÉSO•ô n êi ø5°ƒ»´y¶›O1‘†møl¯LΑm³•3{ ?‡›s$Ï>ȘÄa$Í¿ç>Û G4óÍ;´![("üÒ¶Ò¡EKð:ÍSš¡GyÄ‘­4pÃ1;_1…·wŒíkœÂC 8Šs|pîFšd9ZÌ(˜Ä~&6ÞÕiìx[9œ40I(>šöØ:J˜¤†ŒË1>ìššux˜t|,¢Å!?8eæž7ñKŒ’Í¥¦ëĈ#;áp#|,û‚ àu úqµ¼ýï}ïŒØ˜„¹ÁصkW±§§gyEÑ]J©Esµ­­ˆvǵ#1ò>Ⱦ²ü˜Nœá|’%(ŽGMéÒ½œñpãËelƺðjF9Rî6?NF\¤ùeÌfbteô]:øšžgäH'*‘¼kÖ½Ñn¿OaÒŽ<ó7)ãS…Øû¡Eø˵ ¡Ab+yÊŽ8_I#DtB)õa±Xo+åˆý2Ì ±±Ë´ ‹˜­ZÄ­+(Ž×ûš¢®Œñ¨D¼Ž·•‡•‚ÒË ,w›eÓ¯¢Åñ—2éÊØÉÚ¥^/Ê‘v¤H+uH'!VyçéT!” SƆ/Ì&/$Úä‹tÉ#::IàÚA5"Æ+àœ%Q»|ùòñ}²„›@ ܲˆW7ìzçžE=wÜ«}¿J€k ÅG…9ÙÌ鶮HCrTœd1W£ñû؆3 ̹uÂBšeGÚÓ+Kùbä½ð„™vësâ 5SnpE°­HñEöy¸âÅ#I’ÔÇ.AÚàDÍà1i›%Ìð¡Ü1²Çî+™$´ÂÉ]›ßÞoF~Ípì¹êÈ8ÞTv˜yTx?,Â;ltÒVxÍLû~ÃtÂÔ$2²r¦íkVš´Í ËaK]¦2šD•5ÝY¹ÈópÙ!_ Iâ;Î …â@²Õ¸>u8G‡1&J¾±÷ÓßþÞË/úFíV†#׈˜Ù±cGØÛÛÛ³jÕª•J©»‰(Ÿé›—²HVû“2¾Ùÿ»Õd„E|;.#GxâTÓbòÍ›£‘†Š0¾ÄµïüŽ®ø“ÏÔ›Š“ÖŠwO‚£‚`j[IÃ<.*NK‰DmK1#FríÚ#!F¢Ä—Äû6râÙζOîãqð[…’{ÚR y‘m;ûX'÷~á'ÍSÎØêä¾ËDæ=çS} q‰Ì1äÄEr.ïä‡$åœÌ{"»^žËàraF—}«ôØÏ‚©òäçölrÒļî@ð)‘~ŸP½¯¨öë©zýÌÿúÞ¯Wú¯Þµó®[j!G:@üÁºóÎ;K]]]wÝu׊®®®uñª3A QiÎwà.Áÿ†‡˜ÃVÃï:Ä„šŒzï›Ö ¾DM¹i|”Î ×±Þ²6ëß Gºzì›b·öÏüI•‰ÓÑÈðOç9r=$x®s»P›v$X»}×B†Ì&—ˆ$ÉË’‚“à&³B“ÉãE¯ѹ˜09uôèÑKýýý£›7ož–Up`n V‰¼ü—¹à®_¿j¨wè>(©G€Gq=õ¢éµ4Jåmׂï[í*j­ð–d_v@öÓ"Erm ¾ÒŠå°ÍÎuvÊ*Æœù·Ô¥–ßëÏ?á&”Œ“ ä8°Í%™M˜’1FßÌd³IãGÕ3Ôž¾; Ìêñ©Grîü¿2šõ€µÙGŒ5þñ6®cu“³ ½Éb Æë ÞG)MTFÆË¥±–8ÿè缫ãG£tÇR(”8/'[·5‹’#ç¶r¤ÕŒR9•Wg§¸^UÈlÑ Q‘§ñýíÃ$*œê¦±ï¿ôÒœ'K„aˆó‰ ,œžžÞ¸hÑ¢8—ÈJ­u"†s¦‘s¶ƒÝ.LÃG¨ØÈËQÑÎy6œm ÔT½wÛ$-x Œ,îd¼èx¼Ýt~¶™Õjz¬¤R`¼¤Ü™„ÆŠª§õÏ9’kTµQ—ØeÁ7ctƒˆ~_A0«%wóÂk|·Â­J†Îä,¦ZL–h­'•R— D—7V*—–•zOýääÑ#{~ùË©9ÐT@ ˜÷ؽkW±21±vmOßÎ0 ŸQ*¸"@I¡áÜ‚ßB·]6H¾ÃÄVœ±ULæÀ"# !­ßTpêÄu’MgÔ  # ]‚ GH¶%ΤIT˜-ÉË¡àN¼{ÂD4w¹×h°MØ—#³ Éø àÚ¼¼Un¨‡›"ïæu0¶¶˜ tH0ã&ǹÎv™×–s |2Wf8ÄŒŠÈÚÌ·Zö ]yßØÛ¬q0ˆ ŸóîÛÍciÀ ÊlLž¦ê_¨X×Ð>!O “.¹ç2‘éñÉÎô: ]ÍÐ)s×zjsTþFˆÙùS߮ϽvìUbŽ¨‘hZ#á%Mp\CýØ©Ë—ßU|ûLJ‡?ziŽªJ„€ï|ç;å+W®,-—Ë÷‹Å °˜ˆŠ8“&¹NVŸ¸Ñ 99„J«°ê<˜õ¨`²Þæ[<œ IDAT»uzäˆË5{lV¯ýž*ß1Vc¾·2C#ùÈc¶›,7éêè©èÂÁ}s=!k»°šNŽOÐ.솗ë$,‹ïŸ-1ÂÑnåð<ß³ ‘™Ë9H:Å€ ‚åÒÚe…îme¥ u­ÔHÿ¤ªiŸVÑá]þç—eõ@ ˜v?õT_WØu×@WùÂ0Ü(ÜDDÝ:ß`Ÿãä÷¿*ýͧˆÁ ¡1üK ‘–£Î½ÎòŒ™öœ_¹‘îƒÖžˆß „Œ²ÂJÒ±°ÕÜ^IT¿e®œc6YÍi~´œÑæ­ÕÔ–BÌó8 ƒÍ÷6È°¤J¯vÃìÔg÷H^HGÖ`¤€•ïÃ&!,²ˆŸ‘d~V€ß^hÔÉÇ͸;Rƒ‡}_™ÏCéø²ËºåQñ1÷™Q‡éµp®E†dê$+Ì ?Üë=sÑ毲"÷F뚙ͶB“¤4C’|õñI[Ó!P€FkZ×tôãZö•ç˜y;“#¸sçΞeË–­QJíPJm@Ä^"Rq®°ˆv9náÙå\§·UÙ<'»j¤qÒ¦±áD½wë´î™Ùææ2½Í?ü„ˆ9»o)7²ö¦¤ÇÌñœ}7_ ”-»FP™8Y~u®’#³QüØû× jM„åíãen„òÄVcï‡6ä§8É+ËiE´Ì%Ün.–­-uÝÝk” ÊÌø‰_‡WIÁj¥ú¶Òô*”Õ[Ÿ ÿù÷¾W™³‚Ï»ví ~»\ +ѬP}–"Q˜¥’Èœs#©¨Aj$ù˜“â5ÞÙì¿MXßÔŒ¿*ÇæÛû`kXìÝP,o,–¶…JÅ7;3ªÉ°{›s5B<¤^­øêÞ“Ÿ|ø¿ýàcŸ['`Ž &Eþ°RY2ÕsÇo£¢oà—Ä.j¬ñï! 8ØŠ(Ü‹˜1ÏÌ,cü}áʇ\c„“%¦²!-–øÖÄ;æ*Õ˜XJNÓÈñ`œƒÛ~n.› ±‘ËB2çÓÃB“TIw}ê7|% màªO®g¢,Ÿ0rå&LgÐgYu3‡Ý9ê(tÙ’¼`S5‚™¾ÂP ¤õñ_¦al+ÆÍ\7c€L&ͼF¼]~š {rŸW‘”—X™ õ)<Ò&»Äƒ1Æ®Ÿß|þLb!9>p1ÊQFÈ9d#Mœ{ÆYœ{ßTƒY¼ A¨€A@%ûŒº8éÂH{ÜSÊKMº¬¨ñ2&Nîz >BT¯Ö_}ós°1or$žþüîw¿ÛEÑfD|×FQÔH°j+@ìmìóáV"NZ‘#­È‹¼r¨ ò¥‰yu©`R÷m™Šz~+QŽ`ú²³_öƒš½H“úJýK ŒW«ñÅ‚²·MØ GèÒÁ×ô[­ÆFù9äW©ÕŠñ•mwŒ½-~Ž¢(J·ùHßs žP™N´úˆ»Læú³¿ v/”—­íê~°¨ÔŠÔ v?$3`<î8A3²,"}E¾Kˆ¯P ~¢(<ôƪþ+»wﾶ(@ ˜‡ˆ'Þ– /+ÊO â7À— ‘`5O9™ümæÖ0í;oF¶Ý–Á³ÜÜ¡Ëû¶w )s„ì|9“_n?À“H“·”Á˜¤r&Û Ýs¸3›‚Â'û\\}ÀûÎ)y²J­T¸vÎ £q˜µÌ¶w줫Îß6äë¬Ûù™ãšdØßvvï÷$›@´nC)ÂH;ÿ °ã9yg’!¾›‡Oœš!9.ÈmKÕŠÿ‹iÃyç׀ݾ0­vìž!ø`õx–öõ®v“©Í½Šÿ)=¹KØ@¤õqXùfÌƛ׊«ƒlÕ =Ùí´ë° sêb¾Yú¾€„Î#Mú ¾‹ ^! ?)–ÃCoôß|ó–&GbBdÇŽáêÕ«º»»×…aøÖz5"–x¹<Ù<´p„fãüÌw•‰ïÃöK¯ÅGÂç· µàÛòœr~¾l_LŽ,Ø<õÅòÕf]ÆR'Ç­³_å™qÑ G Yí6[òUOû¨2y"º<üê\[ÊZ:íÈ*èðºùÊuºïfÀGltª2Ïñ­ò‹Ìõg¼Ç…þ/”º¶õ CTÝÀ%ŽÍkãÏæU§cÈ>šÖsDˆšH_FTG"¢}¤áWª¼ýßýèGç×®][ß»wo$ùJÁ­„˜Y¶»V†J? J}w Qo6ã&Ìô.I›üÝüoâܤ‡w6;Žø¿)ùÀʈïûÍjÁ¬<·wÜï¿Ù2î&;¶#3€Ûx`ÚzyRügêè2UCæpq•_xbçf°n>N®âÃeó›6”[Ƽ€)eïC‹ àcŒB²Ç!m‹!oÈ'[Œ{ÅÈ/bk¶*Èî’´Ìc«^ùö#ö¿©ñÎîgÌñ n…;ÝîuC6f¬[¦Z$ÖÃè¬c‡âøc<|‘>{îuˈò €A\™cN8å^/;é-™ÏªoЬ0(‹¼±‰M‡TMšÕ´1ª#¤c3øU…&ošyK’#ñª3aö,]ºt)l€mD´$ÃkÉËEÐΚ n×eÁ"RìýöG¬„].©¯¥£ŽªèžMãº÷+@X0êLXoëež>°Ž‘APêŸ «!â³–XŒ1ÖQuâ”Þ;É‘Vh¥Úé„ËS‡Øè” éD•9†ý;¯¼½ãVy~ãôAqk±´bU±ëžî0XŽ ~ÿƒñnØœaôL&4Ê*f¼ó©µ8뜠÷#P¿@¨¾qèâÅ£úë__üÖ·¾Ue‰@ ˜ˆ'Ý6nÜXüîúõ — ,½+ ôïh€§‘` *5Λú¦ä¼[!gâaæSQÁ=|[…«,\æ$uLŒfrÚ`ÌìCJ“3)–Ýæ$aj_ _ˆ@®ÚìþY%Ìê4.~®ELúÛJPköɱÍÙvÏõ°C“mFâU~&‹H³óUp刡"äº3;×P&’ÌÎf‡]ÛLðÛû·È?ÆwdŠTv¬EÙcïúYb[®f°•Dɘx8y6.Ë…“Ýæ½lÜ׬ËÞñó6È1®y3o"/Ç &°îw›¼òO~Úcl“ yÇ1ÌT€d=pÇçÍlxî—ærÍ©ªÙàÈ6&!œTDï©_D ß8zñì ³1or$þ`mÛ¶­°páÂþ+VÄ*‘»àNèk"“Àç,µ’Øß®hE|ä©>ìýí>´öö¼¯½ 1P“Q׺ Ýÿ;Xp9öâHžè´-Í7 ϳ„Õ`šàÈ„÷¥ÖüHéêäéèÒðO?/r¤ñd—MЎظ^Õ6. bu&廃VÊŽíBcòÊqÌö9ž¯ ˜ã±.ƒ –„ÁÀ–Ryý`¡´y&·S‚@F‚Ì<Þé‡t†)¹È¸w2#Œ› Mc<"„+ŠàCBx£Fõ×_üÕ¯^ÿ›ýû/~¶£!׆X%ò·ÿîßuýΖ-˾°lÅ£ETÏâ#°4¥3¼+jÌÀ÷ 6•¦Ç㛇M•l–¹çbåÊ°¦§­÷²Ág§eš 2Þä|æØ7ÄgÉÙyFÚÌØY`®ò§Œ¬ª†ŠhÆpxr'€a—ØN©+0"œ2ÄÌ!ôÙ£›É±¡˜qi¯~à £l"ÝZžÙc¼M¥·wͶ8÷oTÎXº›IGM%BÖ.ä°Ø û:¥{lõ‚{˜­>22¬ä…s÷[‚Äy~Y¨ ä\{›³$Æ4𡶉ðÝ'öì”Q±û˜8µ@ä€A‚ä‘,6IhtÚ|•xÿζÙíãφ?¤Î<,Ó >D€7êúúmÌyOŽÄ¬={öt ,^½zõ:DÜ‚ˆ« +)“'—÷íƒyGnwb<äˆ_ÖYrÕV/•vN¸o?"â¤î^;õÿ^ìƒ{½+±cƈfǺ/ªRßb(öôå4­Ú£«“g¢ ^™K YÛ©=ì2׫ú0Î €½AP5¸ TÝW"¯MN‘žœÔºåY”mÐ aÑ.ÇH»çz¾>÷ÝÁŠ°4°¶P\¶¼T¾+Tj!Ìäâaã˜Kž¶.¥BÒ͆¹ã¬*ŒŒla5Î|ü4Õ¢èB­Vûþ§Ÿíûë#GŽÈŠ7`Î"V"]½:°sí†Í]åò#ˆð$ >/ô•´9o²ìo*Øh2›3‰Ï æÙ`’JN]9udã‚f=ƳÉÆÛ"­Ìã“IZSé“×^£ªV³“(ÖǨX‡¤Hs©jcyÊYámÞú< 3ÚÃZÈ’ûÚ¤vÏdwB&ãrclÌðZš ˆ3‚=ztà­·ÞZsÿý÷¯U"ˆ¸„ˆ ¼y¾å4ófžóŽIþ½Ý‰ð|ø;-÷¥ÌnóÅœç _ âh6$l.ÆÌAö°“¹•ì ™,µ§‡9¼bö¢k,çý9¢•\о&y„V+´ IÆ»ÂÅAaѪbaÝ@X\ vGD“­G¯FÕ‘Q £c:êc£D¨[­@c#ïùl•@u6Äç<Í„ËVÜ],Ý· X*(ìâûçÑ B(ÝÊ Ž4âM룡µb¤‰y ?@¤ëP«×1ªG¥¨®kýýýZ#æ$vïܾ?2±úñ¾‡º–­þ-x‘¾h¾KàËÁú¾¥v^Ãüç$4'®‘íg'âĈM0'Șd6r4ä9.>Ëƃ”è1?abõ‰;]¦zƒ«@ÒãRßÒ×`FyŠÛê³Z”Ž?_æ´µs‡Ic<à!5YBNÿ¼Cj¤—3Xe‰7›‚ál3[óȲ“4–O²¿ÓÆQdÞ®vX‡Õhƒ.ÈœZ2H>>»‘Ò‹oNnò†d„ØL–<Ö!ý,BÊ"³ñE÷ž:ø¶¤.3‰{ñì¹$¦ÎH®QòÜ#¥d:cÍ F²±Ü:ç—Æõq:æ¦4LÉ"j8 6sî¦ÕØ­FçÉÀ$Ç&4‘2óÑjŠ}Ëgçó4ÕDZA]G ³E¥ZíúlÌùFŽàöíÛ»zzz§¦¦6mÙ²e".Cg(^;>¾P-dùíVµ°ÁÉ”<ˆš$C' `=/ÓJÁà;–o5"Ê“a%ß0÷A¶¶5^>ºy>`Ì)6^ †´4‘Àr96_›sì¶h •§îñ)NZÕ“ 1\„}+ Åõ‹Ââå@-LÞ91ARD5Ø„«#„j=Òããº~e<ÒÆtý¹zmä ÀT«ü?ÐF-"4(é„„™Oˆ?Iª^_wzj|ua:€žB„è* Œ‰¤æÎt±¯lóC­íy×nKÛ³€Éµ¨Õcb¤ËãMS5MãëÖ­Óû÷ï¿¥Æ] Ì_Ä“n}'Gû1˜¸sá w~mùª¯(À»Iá2 *¥ŽƒJÁ'8ÿÑë,{XŠ™ª¹sÈT)L§¥QŠ2﨑í™dƒjpƒR2„oGk_^ت==V7CXûx8Hæ r¢ƒ/wš1æ-ÆýPv„sæÛ˜9 C“cÀ¾®æDW-¤ãîsBÓO­¥ì0rʘ¿³ãLûŒûßÈJ’U¿éè‹ X­4„:FÙÍ̺˜]‡„BãšÚ«ÜeϦãƉÀ¬Aæï®Ê’ôCám{òŠøæ2•cX™d,9yb“9·DûÙÆ&‘F@F9õ%Ì {ŠÒ{ŒR˜Ùq!ʯ0žùì°{ɹž>Å{¾‘’­šÊžÙ„DÉÞ?ìžÁäÏ¥ùÅ1ó/ecjÝ š¦ôuÚ˜ó‚‰?X“““}Q­Z¸páÝqØ "ÞñÚñDN’U`/Ѫý;ïØv¤ÌÓYå›VŠ[>i$­ÐI˜ 6„#܃KÎfhü…ÎVß0®¬MʘmÉÚ˜ßC:¡n¢¿í gâÈ3ÖÀwûÓ€£Ÿè+ Âb.XS(Þ9T,l.©pÁTt1 & Â@©îRPZ4ÒÚ:Puu¤/Eõsc::u®V¹ P­hùžÍvKîÚÏúlÃåæ㳎Z«:`Ï•Zêp©V…Š*€…ôŠÐS(B1 ˆÇ§y-uóÙ`_BÃÈÒ©aœXy<»=¦fGƒTÑQüÁªÕDZk0Y‚ÊÄÖ­[iÏž=ŸÃèA†ç¶o/ .]¸½¼vÉ3¨à~ ˆW6ì# <6Lê’'ÜÈpR¸“ïi̶zHc2‚͸›ÄC–€ÓÛ| ™˜§ãNxhHHžO€ ;‡#-K®Í xbL> …Yß B¥¹ß$}È æMט©3R§;CL¶`b—0 €_Ä6³CCÌkš%öLj-Z’»Ä®gEÃçKf/©š’M”:õέÜ7D3_²km“VYk5›t'°kÎÇ9;—}ðð›ñ‘AŽÉúŸ*é‰ ' Ï·’„´"FؘóúœþY¹RÒg‰Í#rÒIQ_(Ljñ¤¾Éù”C¤ô‚s_fäD– ÖÜ—öÈÉbß8Ä®™â:½®ü9â·²Ëh4ëH³±‹lùUÔì;»O³{Qd&ŒNŸÈDíü¡â6fµIŠ4lÌ™‹6Y‡ÚuÙ˜sš‰ó‰üêW¿Z¬”ºg```,n.ï²€­œi Òɪy¡7>t¢(ÉsØn´#4øƒh+|ŽvIb«M|Û´ÖHJ)ÔÖ§2ÍÍf šË–Bú Bjè4+Í%u|³C¦Ás3ñÛÿæ]äVí1Úæ!/ìkÑŠÀ²ëi¥.‰ÿ߯TqC¡´r(,né Õ@Hœòf¬súÍjŠ¥/†€ÅþBØÓ_—׈¶®*•®T"}uT×N\¬ÖÏ_@Ÿ¨CN¢Th¡ñ½ :y&çÛsç 4uÏDP¥:LDu¸0=¡ àŽ0„¾b ŠPñ•Íœ$•1‚$1J’%A1cD§÷EEP«TãiözˆFëã]SŸÏÈÁ â|"¿ßß¿©¿«÷‚¿¦7à$*êæ K^Ȩwr‚9 ©»dIñ}2ôÔ½²B2š HÏ“~CÙï”!£¦Ì±8Z1I×ç<ÏÖ‹Èú™8¸-&®˜ãŸÕ9ú¼P6nY;¹BÆÊc‘UfªPÒQ¥ô|I3‰lÇšÑöôšÉcÓd²‰½h,«ÊÈ2LÅ”@¬ŸÉ¸dY¾æÛÇ\Bæ sz&½êFþ¦k©ÆÃà\Lb˧^IÛmäÉZš´ÃPñ›8u–3{<é?`Ú/î› SIcÖoOäðÉJƒá÷/³wòJ`×8»æ3´Õ AaOÇ߻Ȟ+[Ñ… 3c!#J2˜ nï°o‘†ì¾ç×ոØÉLé“©;L5?«Á_5û§Ø;Ž’ßÍg‹šä¯Ñ&`+"Z6f¥ZmØ—‰=Ÿ^k¤ÑÒxñºlÌ9IŽlß¾½°eË–E‡¾o``à¥Ô¢N¾Ü" ]HN'+×ødü7bÕŒùŸŠ¡•ÓîFÈ :ǧ¿U¨ùÛ„ˆSJ˜™.Ì’±$€¼ ­Úl{jdÄíÔ ?Ë´#¾q°€Öeó1¦JÅW® n*—,/–ïéÂeaCÕÕœhJl~g•UúuärÔø?a1,aWOK©¸ie1š¬j}a´VÿôtT;{êW&´ŽÀC„úžQûw'ï„ùô|&} f˜¿bj”Bf8é&ÓEu8Ep¡Zm|°ÊJAo¡½År#ü¦†™\Ø0„’gU¥Ì2àªÕ*Ôãb!hIûf.íøŸ|mâµ—ŽHÎ@ð™ãßìÚÕÕ¥Êë‹H„ˆ»qCLˆ4ÌêXY‘Cp¤d†=ÙÐ,c[†SjNù6ù×ç@° 5ççËú‘Iñ'+ßyò¯’õÕã¬6Ï\ Âgæ¹½ÐÌÅzC–}Âþ6ò¹˜ÎXBZ`6¸ÞÙvbäJV¿}] oÐüm2 YW=+Ð89E’í‰b„+È$P R€·' ËhvÖtª3"(ÝF¦ŒŽMˆæ½ë±Í{Ê=_Ò1›oʈ‰„»À”˜bÂÔŒáĆ{׳~X X¸mÏW{vŸ)ö0Y‡uX×—8QÄ’¨Ú“¡Éd’1Æï54úl¯êƒæÎŒTÈ /y—Œ¥oŸÇo`rTN¶¿cìÞL‰ FŒ˜Ï{6=ÑEÙng±bî¤ÍÜ·ocƤHF£“E/ÓÃÒk1þç×icÎ)rd×®]Å Ap/"nGÄÁ$—ˆ­–ï„œ¼yŽN^=­ÂpòºÚõµSŸÜê‰í\Ë1’lg{+ÃÄ÷o~p¸—Æ·f³ùB0ß…ÙÀø.ZŒ¼ñ"óÍ’²ûsx× >®y„’ÑþeŽÏ˜ÊûÛ>¦¤T°>‡Ö”»ïîF\£”*Kò•½„“7~óÛfdÝNÞl\œɈgÞ3Ù¬  z‹aÐÛ„w.¦Òd…ô¹«õú§G+•ãg¢Úä„Ôu‚Ù<‡sõ™Múã2C‰HÉœ\jHÇtãõœ™žŠ¯+ÜQ,A© =ÅŠQŒÙb6ÉŒZ¤AŒÔÓý3³lb&ìv‚d•@ð™"&E gƒê¨~Öbq¦ æ÷­äª˜8ûÉw2gž{Éw‹ëW!]"=Aæ8&H' ™“`0Ä ‚iwðy;‡wZ¨™LÒÚ}¿“êý¶E2ÏVL²ÛÛü˜aL¾Ÿö•²\ i˜¦‘”–9Ö̃ÍìnŸ«ÓÎcF·™â™Ó˜Î웞q:cÔ×ì¿úÁÇÅ翧·€©XNëOÿDÍ‘ŒÏÅÂÍKc¬ù½õŜг'Ç{ÖôâêL‘ý›9þ–šÇ²Y—ÍßVMBÙÃÌÑ뚸¶¶‘t•ç_IÛG7mã:±ºÓ ò”õ‘$ÙÄ‘<Ö|t˜ïŽ'š,\ê? »îÉsh °íën*S(ËEÄï÷ää黂_otîKÞÖD¹Å3!%æÄ«÷|Yç˜/@¼)ÆÓÊƬÔflÌt»A6¦£~Ý6æœ GbR¤««kmE÷À¶8ŸH' ‘NöïvDI^Ž/Lf6!7­Ž™ÏĈí„'ð9ãyvò›oçåmƒÅ>oz¬®Ä9ÓR™ gͳ/]–8(‘ù3ã‚ >†}\Ð*c:.I}Cà#’Ú)FìÙ_9ß “ïš$ûKa¨–¢ê]_(}a XØ¢4 fÆIパÌ4ëÄŒ;âGcmyklS)2760Tª'µ®¤‚µŠ`ÿ…ª~«N¤µG-æËC×ñì͵gÖ $ TvŸ@ò1áì^"ÓfÉðâ¯JÁ¹é)8?=]A Âzc¢$¿ ‚™þ'—:É-R £I>®š2ã‰ÇšÁh†“Ÿõ ‚Û1)òÿ³÷f=¶eÉyXÄ>çätï­¡««ç&eŽFK”Z Ô’ePîaA$Š2(C0 ‚ü ¿Óï–l? $‚L&l mØ4¸)Bݶ¨&«‡ê®ª[u§Ì¼y¦½ÃØCD|{ís2«²ªï½u¢ûVfž³÷ÚkÚkE|ñE¬—§š¨þOÚþ%&þèç÷Ǧg‘0E†;k¨ (8€{X  _ ñ¯_ØÀžÜ¤áß’ôdÄæÁ‘º ÊLQšõŒPX4˜Ü¸ÄË“ñM¾ÿ¨“ÄÙL±‡þôy þj¤¦ÓD$ÔY§PÇÈ°P‹Ûr¦ŸpAOàÉm=²qã>"h¼ãœA½RÑ'dŒÀg Œ„=,"ÀHáD“TŽ£-`ìfà,ÄH$ý öôiûâcÃD@ýÚ?æP? c’û\dÕØ[d:&&V¦¡¯+Ð1crXJ Sž·anh?3™-3¶p<Øæ¦Îßdj Ÿ!#ð9Ž©Ž›ƒš£Áa-ËsÊôÍŽàWÓÄ÷\ ãŽuµuŽGK‡K|¨·Î«–)²Zo:Sm ô –g IDATI§üÀùÀ:æ i“¬®V«³³³³ŸšÏç_©ëú'úS?oGÐPȆQ ¤˜>J9Cö…ê\‡Ò_’ç‘A2e„çïJO}Vb—èçxOcÚ¨«Š¹f͉I›0ÏH÷ÞKsËa ñc»(n¤Ã7£¥Íê^I›ú¤´¼o)µWÿíRJýºk\°ÌU³çù~úøäŸÍgŸeâÜ_Ÿu{ò\¹î¡;'(—°Ïäᮩ:J]Ýç´¨jéP‘îÞÒ»¹/ ë‡É2ù($·uÞ4GBÍŒ`|Mq×~6§&'ñêvW ŸÖ[zºÝÐ;Ë%Í+¦;óE—§äÞñ1ÌfÔlkÚnÖÑ;I½ç´÷ˆq§ð Ìç†IÏVóCΑƒä š´IVÿƒŸû¹×êuý ó¯µGñŠT¯Åç vÛÝSrT“d ÛµÁ+fd>Ë Ù K@x‚a'’ªDh`r“!ÉHMO_.‚3 ‰!à±F·C CC> ‡8˜‘ŽÑნæH`˜P,§ ~zyFy µ œÙaÏÞ‡0]𵱑‰ƒœ&‘𪅱ÊãX[ü:c %@ÂômœëìãF<`ÂóÓó°úÙc1;Ž¯¹œV·]o6´^o|~èZc̵:S>¸Žù‘ƒ#m>‘Ÿø‰Ÿxåèèèg‹ÅW˜ùóDt\2ÐnS¬(OQÙ•ì¯Ù¸”ä¦àÉó S¬’ìcš”>ÏÌ4øg3¦mÓ4Ò ÎuP_STD¢8‘"ü`PfŒDŠhÚOÚÜ¢xBgð£6•’)pŠRŸO•;õ|½·9¡Í꧟|®®çÕÉ|Ñ¿W8‡”Š…R=' IÁ"œY/B½‘ÝžØÜ%b’îP ö·­tŽ¾Ýály Øuí.y–ßÕ.çH]ŸôLOïÿŠ-{7­ËÍOV‚ÇAm¡Çë5=^¯hvAôé£ã.ü¦¿ ã†û§©Ë]G[ªéI-óCXÍAr[•6/ÝßüÚ׎ÿäë¯átqöçdÝü ù2óÀlLyL82\\JT~LG1Ðì^cðäi¦Ê“;fFÆÈ0ê9„Tw‰†hhWÁ’½áZO+ Ø/ª?¡s£{~Å#½ŠPGáØ&I’„°‚¶ #.1"uí6à3;=ȧFÖÙ5£¾"õ=£!>¥c† ^1¡§9RÎËxAîT <$|CxËÇ™¡,±> C"¥Âå÷A w†™f*‡±ðm¸Ï@#ÌsÃÞ7„ ˜ ’Hzo‘5FÞ¼ÏGÀƒOýLÒœa¯t )bÐúàø`ÆŠ ãe¦…ШŸÇjú0¢€DSe(›Øoõü<>ϬÆL£yœÈø~À0–‡5Á†ÄÐ-´|"Oߎ6|¦F¶u á‚©•­9d')6ÌÛJ>¸Žù‘€#í©3÷ïþÝã/}éKŸüô§?ý%"úc"ò9fô8ßžL(Sžâ)0d×uùïëd_/ZÞ‘½NeŠ2Í)ž’Aêî,_ö…7I °…¯¤ìX¼[Ú’‚÷"ÆÄéfÙž›s}x“kJ@I©òõS`Iô(dihÝHõîÕ¿×ç¡Óù|ø·èNAisT´ ‡o‘  èùÈ:Ò‹‚ºaõi†¹P× øuzÙŠ¬ô~?'LÝäïgQòš·hs¿ÔMØ4]ÙƒuŠJ‘«„ *‹-U×ý}³…Ê„ œ%ê6aep ˶ay|ïµÙšÞx®ºø 9È3(- òµ¯}mFo¼q÷oÿ¥¿ú“ö³_ü \ÉBÒü!:ÊLÊIw[éõÙÛ?Ò5ÐÊÖ¿ðpFŸÀÜËÉŒNƬоžN“ Ó‘u1}®W?€ #¦Šn#£=¬Ð|bЇ¡0ä„õ>»`¿-ëqn0ªŽ›0vœŒÂS¬.ì׌B©¹\.Œt)ܺÊ'ßѶ´%´QVdÍ+o]Áá~GabY…¦†Cnï.Í1®GŽ¯³ðvý™'™A‚º¤Šq|!b%ÖÈœ5< 58Î%ì[]â<‰aà0’AL°1Ü)Ð7«$ö‚oêl GÀ©/HÇ¡r Ìš'0pl†VWi®…¹èàK(CóæØ=1Yn¶ºèüjuüÍvÓ…Òh²åѬ“>äÎ@QŠ·µ|póCGZ@äüƒ°¨ëú•o|ãŸýêW¿ú“"òïÑ'Û$«ÖsUJäœ`¯Ý•Od*áê®0|Æ‹xTèud*4cŠéPú¬¤Ðd:*”-IÝR|màWl‹c!Åè ¼À¶åêæk „ÔØWc æ6¤Dë<±±32N屘R(™¹O:\³jjZ®jz¼Y·F9qÕ$݉'UÕ娘Ñ@->žm;ˆ°+œºè¶kcw†¹ L8ÎKuÂЊ¹=Ìj@op<ï ®ã­iÎX¨TV“çæñ×ÙsÚµÖ ÔÆ|6ýͺ¾ŠÊ‡=J#d/¥%Ÿ4ç'Ëå–rƒä}Jš}ÿ›ß<ýÛ¿üË_øµÏþÐç¾ø $ü‹DôS"tLaUš2b ^eBC%%\ÅøzIå:žb׳[nš¢“Ec³Q—½ÐHi÷|nœÄÕ7,ïƒ1™Bž)êvKÈŸ2î;³I-A*0`­žu¬Ðpƒ;õ2SÂdÔC.h´½QìÕ«†>þÕÀîÀpI! XÝ?G:ÒpJ †v¨WŸ%÷L>¯2&ðŸèG€’á „è»p&IÏÊF82qð‘îá÷öE'o†êXݘ-ÙpvÆDptarc[s{”ÄŒxOd°°Ÿ ‰u¡¾½)¯¬šàUrÀF‰º•½¤ãë$ß ¼Àü{%°0`dIèkX/tt“§¤;Fðlÿ®¨{ÏËe´!Ú›º¦m½í’¯0°Pùfxmþõ}³ÝÞ‚Žyë E ŠüæoþæËÿüŸÿóÏå+_ù1fþ±ªª¾ "w[¦Èu¼áTv…¿ÐÖHNØ:•»dʘú8Í{‰ÔÇqÖ)æÃ.FÃN/·ƒTyJXxI×m¤ k„âf¡ GAqTP±òÛ‘ãfk¤ØM{ò•Lõ±øî\a©²Ò&ò\I{êÉ–æ«%U3:žÏŒUrÔ% ¬QD‘g¬UÝ‚!u.òÀ¦K@O_©º‘ ½  ã.)…¶ØUE|Ä ò¦G-ÏAJ:ÜŽAŠ4A˜A[îýrP°ú q­„–õrù¾ô¥šþÍ¿ùè;ì 9Ès-_ýêW篟Ÿî?\nÿè?öÇÿD%ÕŸ`¢/SÅŸ"‘Eoaü¿ A9×U ½&„>$…Ÿ‘yÇø“bXù8ø‰‡:püÛ¾c2vA2Z±r ãhˆ¤^㦎–«Þ|׎ˆRU¢-µQ{Êõ×:„B‡¾EO»„]ËÄð7äˆöX¿öÑýÎn¸âØÉàÚÆê0s ÏáP¯?áC‰Nì2†ã{)nÖUÉ’tr4JaxñÁSÄ x)Í'²jrËž.ÍšqEFóÛF–Å÷¡ž^£ñ t̸þêv 'FÔÄ´Žåà©L‚7øh:%£é/ЩíóP‰¬Ö¥YÌ8l†x`U3Ãt´L£‘Nz…#Ì3šð¾e,Ù)­^Ù²D6(ÒXÉ®2óÜ%{Q–¼™}`ó¶ÀþùŸÿùùË/¿ü©o|ã?ýå/ù'ëºþl{ê 3€H–]á1»rƒdeßq½S×OÉuO»9ÈXFôÏ´ Ÿ 9GTô5ó­ÍèS,CÔ3á1]VdL–æúPPz0!*Ì·7éqÁ.SÌ‘) ©Ä&ÙJ­ö!UüŽÜã4ôI Û¶p›Ðó|½¢ùlÖ%'³y—Èód>ï>›¯H?-S¤²R£·+Ð[õÙÌͺ ׸ÆIT/Ú;WÌ‘$Âs¢>¹µn’‰Þ‹ž…~nÃ5à€‘6|¦‡º1°iÐ3ŒG6¥Åß´¨ÇH$zuÕlýá?úeùßøµorƒ¼Òî9ÿÅ/ÿòéìÑÓŸúÒ+¯|íì³_üªøK,òE®ènoÈ „Ù‰OSˆòŒë˜ÙEhI²¨⎃:£ û×ÅlõlJŒi@&ÉxO…ÊŽj” GÛ l´9…_fH@?‹ì†œ3ÛéàQ¬^4T#3‚GpÝcx"œÍ[ Å\ ^¦öÁ-ƒ2Èþ‰ƒhîz÷ºk )Ïš÷GdG@ÆŸˆ×7<ŠâØã S\{-„Dľ0¯fÈ7†õй(á¡#¦RÜ$ÕÕ*íöp®²jµMŽ…„r%ÜGT å/»1w‡áyΖZà)Œ¦{K+™ HŽVÛm hV8qÆWÑ1¬Èdó†R¿ …êúEP¾– –Lž8=¼‡Ã»àyZ¼Dë‰6]݇Ïl]×b PX8C—Ú]WëÍÓGøËøé˜b=ïÝ»÷Ù³³³?¾X,~šˆ^‘“ªª>”|"ו]§ÈàßSRòÈæ2vå)=njó"È0– ýnSA(-æôAvpx;s¹ÂãrCˆd„"°`H4rÔ»ìvQ6Üô‚`Híð’£-B•ŠáÂdŽ‘þ…4b²N7âÝ0 eIìZÜ‚,Š'¥xN `¸@ûLÇ4CÜÇš‡ý=óˆ œÕu§Ø·p´êfÚmQƒ¤¦ÁÜÐØô( 7l Á6þ>& ÷Ó$„&Ýa¬ëðD€ ?ØÆÂûÃ=€AÀ,Žs–à¾áz=ÂÖõ$€³Ð؇ɣã¯aGhv`ÛüsVVÇRm<|x`tñéDñrø"¬àAd ø˜dpa± RëÂ| ±öEîP i«›š6›mŒ4ÒÄçY¨®£^“ cÚ¬±ÅäòÿxóüO¿õ? ó}ƒ#¿ôK¿tü«¿ú«ÿÞç?ÿù?ED?KDw¸úßme=2v“Cp2£dŠ=‚² $Ùe”½È!Y‘o‚ãéS2þKåTUÅumd}Ï]_кó¼F®/ìŠÜûæÉ©`1åƒøF/æuÙUû€”©PJ}½+|iß³{ÖHÔ6b1 hÀ¬Ï¬¹‹Ð 5mhYoéÑšhÎÝ[Ñ«ÇÇ!TÛ5Él¢ºj¯™N¬:—¿{¤j DGаi»YTllî˜"5„WÁ« ÖÜÀE//ýöD{YøbÙ4/DGä ùÐä׿úÕ»ÿý_ýëæ ¯¼ö7˜ù?f¢OrwH¾[‘º6eÇJØóŒÂðH'×™5@ %ºÎáîdæ<&Ó‘Èôˆ,SŒÙü»åãR@±–†Ì¸Äd—îð™Y6k @$P—Ôf»Ö±óbïpÄĽ½ @ü´—>™wb @2|8•dÊä³Û^ãÖKçL‚€% ÷Ûž‡†¬=Ï ‚c ±zûüFÇÎ °mÀˆûÙê’Á+Xm\&Ì4/¼kíY…Ê&1PŠI 5ƒÞ€ý˜“ ‡¤©+ÙÁ*gýäù| fÓsÆo)…çd`Îç‚Wýß Üï×gXû ^žBV¬SÔÑ=¾»Öç„ '„‹‰Y+>ƺ„ä­ö3Š€Sh†- i"Øb#æ?úShÖ]^]ss]2çÂBÅfëšt#r+:æÁ‘?ÿçÿü×^{í'‰è+UUý43Ÿéw×5ð~Ô¢†S Ø(…½”’ë勲ϻNÀó  ]‰©€RbŒ”,7—Ÿu«tÏñ½(+&AÕ ÈXÌÓ<4çˆ–Ë ^†ë°kv²e&Ê)!»®)•3Ñÿmçλn. JÛ(…M«¨ô ÇFÚpêõµÙѺe›ðlÆÕð¥¾oSG÷î ÁÉ×<ë‚kÒ¬G䤑"€º`Z¦H{¾<¦ËáºçÃñͦŠ*§2úLêú¼"zú\têAr\þ‡_ûµ×ªå措êoT\ýGÌô‰~ö<êÅGô.ÇKiŸc³ûÁTRJÑ*Y-!;úØÊDÉzƒ>9‚"uGãpz¯/JádœðÔ h| XàÔÆÒ>;²ôreÅLKa³%RÕGfGŠ…‚^ätoÖ1 _ ÚK蛡*8gØ3½è÷Æ:gi¹]3‘FO`¶QY(–ëú ?õ±ñd¼Ð´.³s1ü)̓àáwã9†¹ÁlïæÆH ìEïGcv¨g/9ÈÞÀû&&ÿEH ÙÄQ¯ñ±€Êæ4‚™á ó‚ÀH/¬~V1òúL.âø¤ò øô00¿Ÿ¬ÏyÚ²cÊØ#&ŒƒO4¬’úÊF¬»¥òõO§GÇŠï‚ÖÓo=xÛÍ»ÙlºÐùsu&¾7 ‚¤Ö²™hèÍ5ÓÞf <ç[Ð1¯Ž´ °>÷¹ÏÝcæöÞ?MDŸ‘ã›xŸ™5T¦Žû ±Ae_¸Î®ã}¯sÔoéïg]n –¡"“•]àÀð™(ÉŸ¯sŸ½éžLu´ùÆŠ”Kÿ¡ JF8¯×æ}ý´«/¦Ø4û9»®Í×㳇ß;u"ö“ø‚¨k¤%Ãõ[]œÑibwµO'˜-ºw?‡ç´©B·UUs]wwí —›KJò¬½oûÖ/ý¬YpU-<áxS|q÷³= ¨Þv‰WÝãʈÇ67¯óu×9ÕÒ¼ŸÃ]¨Œð00ÌüHšæÉGÖi9ÈAžyùï~é—ŽÏ^}õ3$òx+ÿ9Íf?GDw ŒÐñž0B¦*vÍ°‡X!„=øN6(ã›àÚÅßIuûB.ƒl ë5X­B 2£D ÂÒ-’öMBÃA ;ɽ²Àtƒ¶è½609°ŸH 2T¯ÊcÄ›‘¡ªí·p…X QÝ+ôcýëìŸdl €Á×Â2jM†Ð7æU_R@.K(Wb­½q@P÷ò"GìÀ{ ôÑñ®÷¾ÀÁtðÌÕu›ÏÃÉ<Æp(ä  zHöüKÿ>Ú»rüé{êúKßÌ0n`LÌ‹·Y}âû ºLfŽŒÇGÇÐs²#8ÎoœªËÐ BIã‹j¸?¾Ó>|îÔÕ÷¿”³Ï@ìOÿÎ#ðÝuУ*­¿6çuŒ¹sº-׫Ž5Ê· l,™Gcmçp½à热ÑhÉÄ· cîGøñ?õ™Ï|æOŠÈÏ3ó'é9N*šYS†Æ.„’Wº$4)’ó~úíEI(Y2ê§~ÏžŸ} PÓo" ,¤ByAƒøVê Ó°b¢n†8¥=)ÄÃI€HnÚ”€¡©ö«d©Ô·¥ïð9ùÚR]¦î«˜«!„À\ßuñFï¢ÖAÝ á´å>*be“³têî@­ß5rþ Ì©ö˜äv³²ù¢cè^‚þz¥Ž»·è@ƒ@$D$|¹‘º¹¬ON–yçä yæä¯Ñ_›ýÑ?»ý™³WÏþ3¦ê¯3ÓO‘®"¶†L;\ñOŠ9‘+Ú¸@¢‚×ëý×ê}¶ô±,Ì 0öƒÇ4è.Éxn…´ßCÛr›ŒE3€"xþšDõ•Àû²n×#KëÍs,3÷p1,p¦Û5fW¹•nJΚh@@ìev‚íUì[('¡ c†|Zjo@›Öoä¬ÕÆG¢€¡øçnlcî’dàâ|ƒŸªÏ:¨Nê‚׬?ˆ@WVÃgBØŒiŤŒ²˜(Îør&‡€è;`¹ÏBCÓ¼ƒ³øªî. ËúÃË %‘:T]¯E#6$…ÐßxŠ‚p C#LƒŽÆ½{rÄî0槵ˆõ½sVOÎê§s;¾ÇþRÁ­1WŒŽ1ù¥=¢p½êBhš&‡Âùý }`ö[xú Ú_AÇ”^Ǿs§EðÕ¯~uöòË/ÿ™Ífó皦ùä¾S_že™:Æ÷:Çûî …Ù•?$vÓþšªëó,Ùèò5ù{ü þIuFâ›fÔ²®Ê n.¥…:øGö»¾œ^¯Âñ{;äºí/}Æ)YŠ~·ëûüsJ 1tºð%\C‡E¹bÜ÷ýeÂMÁ6^R%TëJà¹UØ<ˆ†H÷›D-Ýi`Í8È2u4÷.¶É³ S€Oéo&YˆÐLé­Ñó^™v£Únû#>‚ÐßѨPP†÷#™Áà`ðà ›©†Þ´8̬j.éÉ“õ3ÝÙ9ÈA>¹ó‡î/VÕÓ¿õøüò¿\m6?Õ4Mܳ'œ¥ýÕímÁ œïÓU@ †oñ>ÿ/‡zº¨WÛ"8áãüCP2tŸ 5b¬]*0ZØqÿG}@$í³ƒÑ«•D#AœQbYø©eFI¸rt1Ù¨êv™ñ"ã2 aiHêõ®¼üt´®îg(™ù1ÒÃ0ÌœCœÇóÕásÔိ³¹,>TZ€,LF #ù’ŠE³ïéa×oA/³w%êA$—>ü¬ü}ašÐï xI7ãû0蘈‘¸“’ÃMÖŸøØa ½E &4 Þf·;"H¢`ž~؆•/ïÙ>w?†Ð`Ý\Ç ÀE1'ã%w3ûÚå=ÀÐu…$¾iÞ¾æ Ì´áªîš«Õ² £ÑwXJÀ²ë¡ö®1åAndÓ"`B—ôäøë˜;­í¯ýëÛGmW«­×kj¶ÿJù:žu™ªï>6ÇT[¯{" 2Hr(ÍTHO)Tçy–’q^b‚”dŠÅ7U-§ÙÚ9Ðÿá5o¼Ô¶¶¥ ÁCM‰WÁ\$ØHô·ëŽÞËãº÷¾Ÿk¦@*ôya|¸Ë6ˆø( VP,‡T1aX}Ã0ur"Ì÷3]8©_ëuÝlÀÊ}!sYžõwíºá­1Ýí}UÈöð=®CZ¶H3#ÌiÒ3nˆþ™* :~³Y5lh ÷‘oŽƒÒÒ s¨}7›ºytçõ×·f_ä y>äï¿ñõåOÞ[ýðñ#úþ»÷é­‡éñåE—ÞìÚÛ˜¢a<çÀ.ŽûGdM¨ñ‹;:±«ðà'TÀðõ²K ý0ôpªy%`(üþ=¨!!TÕ¼ÃfÄ€ @·4¡BÙP#%À$éAfÕ¹)‰Æº«c÷òðÂjèC_X{£ÁJ\Ðíš&%p´êšq<éZ–\pª‘ÕuLŠß %zÙhõ>yøð‹[ÐUål†y%…9u³@ý—Äz΀U©&Àþ!}fσŸL¸¤ùŽ¿›Ž‰‰—§tLëC}Ñ9Gaüp¬B‹TÅyé![^/ìOÁ°™®”rîÅ1+• Ö*iÄÚ…%ªk—Ì_|ßÂ;ïŒ _|­RFV|]½}³u l¡–ʽZ­i¹Zv§T’½c>l#‡+÷ †gŒ´ÌŠ',2ñ¶©«Gw^§¬cîÍ9RUÕyË W¼­@K™ÍfÝ¿Ì"yžú}žäëxš÷<õ{) çE‘)ãœ[^€wÿ™uÒorMÃ$µ¾ìH‘Ä pdóu±áÑBAz—m®1N©‰ÔS «]æø.p"÷Qî‡Q›wô.ÀÈÔÙH•ž¤éŽwÕ4òæè“"¼i²Í”q<™XVMÔw¤~ñßÔÔÔßnbÚuüöó$ûÖžöûÙ¦9ugcŠÉm»©n:PÄP|ØŒ}l0KÿØCÚŽí"Åd[¿‚"Ü~ß K›fMÕååýû‡c|rƒtRÍø–ñ¹©kz|õ”ΗKZÌgtçø˜îžœÒñbN³jVÞ¯À¤`ŽDÐýšÁ’#kdßyáão˜Â)u¶×æêá æ\¡`e {tÉSŽ§å~VsL O¨3¨XÜlµ½4&G Ý3ämà‰ºú",Ç9F²"á5pQŸ ´G…}Gm=‰Þt6hvЭ(Í‹œ?Ž‹Žƒ„é’|IÁ`‰úŠq+9T°¯,ŒÌʆ04dH¬‚°š¯ IDAT5©miæFœ Œèˆ2ñ&uLPCáÍPÍ5"‚ú„xesŒ¡½³û&å1| ÌÅ<ïrû±õÞŸ½þŒàäYÁWT`Œ’oãŸû4Œ1Ø 5Òù ù†,(èÔdã«àŸ¾ Î"óça2Ü úŒgFÌ©ÈkçðzÕÛ ­6›þ$š ß[«Cd˜Ÿ‚‰b¡_v ù£¥iªÍ­è˜{­¹Ä ®ŠyK“Y.—“¤ý½M²²/dz&SÆS‰MR¢²OåÉÇ®Ó0ò}/º ræ"¹Ù‹Œ-Íx|¤ ¬Í(ë}N½3EÇ3o#dœÀ˹ë‡ÓÜ”ñ‘= ¸ñìê¯,SÀÈ€U,§ê¼%¬€R2•w/ÓðÀ5¹ãU¡°«ÝËV¬ã°ãéuB²ÙVUûR¾0Çòî;Agç÷$§ÜcÞ±špµEñ‡3å™âFi‚Ê*Œê†¶­Cð¶é5ìwŒ‘^a¨ˆ›«‹Ëô¥/Õtƒä ÝéZÕ{BY 퉭rýÞÅ}ÿÁ{ôÇéÑÅ]­VvôcØÃÔÓŽà®­e`ØêOµ˜`ù3ƒÚÖ2pˆÀg¦‡è5! ayû×ÇU#gGÖMÀyƒ¡£ú,dH¨qÕ®å\â©Fƒ_°”¸Þ}DU‰Š¥sÁ¹‘øœªÎùi^K³‰}“DC6eì‰7|ÂEL ÊN²àÄ\;Pó„eæÖÕ›ï} ×Åö¹náÀQ½“‘íPYü9QçŠÆ.%†…1&Ë(û´|˜2~9{îÙËˬ õìðfx.}›Ç«öê˜IÁÎQö‡ûÖ¼SDH(€"8^Ù1™úLï³~!·Esç½çxÑ»(ÞãQ÷ü2ÙÆ3–\‚8PhºîE(Ãê1üÞž¹^¯úü"ÝÉ“y>¥5lýÍÄÓöáó*Y mÞ׋[Ò1÷2G‹Åƒ¦ ¢›Uû³íÌÎs9ës>OTu”Ò‘½·‘Ÿ`Ê›½ëš›”÷<ÊMDyaÊ@AÿækïX1l$¡ü…6+Ý©¶¾ñ÷ȱ¿‰ZN–^]ià×D²dvLn )}_bžìêãܯ鋊©œzì½4u!ö^eFP >ÇwÑB²—§ëcÞ4"/a_(_I:°B„g"§Ju‰°úãy‹SPÆ¿ƒÊᯑ°Íõ*e7åcµõ{<"ë7?ºÿÿû?=€#9ÈA:aâ·ˆh;,äŸ÷kÝÅjEOW+:šÏéx± ³£c:9:¢£ù¢[óØö7¼¢+×÷ 7pýýsÁûlnBò}<¬vÆ\Ö\Ð鉠*‚eB} ìÙÀ²´6DýÅj¢ºä*ˆ(ò*£õ£Òuss¡Žü`5ú#@º‚]`˜Üwvo‹u”÷èoQƒÁ‹ÅVÁ|1û6M3³â‚:Ô{øH jïëœxÕõH}VÅÐã€òg]Ncõal2ŒÑ‰tÖaŽU†šyg–WY€8î)œ{ —  ¦Á™au*Œ&¶ÁË‹ –„{¥ð™vWù=Ê•v0BÛÞË°ðá|fìf ò™ßKûÖB¿J ¢:Oã}hGãfÑϦbÞi˜ˆèÝTÀäy1âs^Úq„ff›ì3^®óÜ›ôÓ‹rrÍûýlJ–•)E®¤€§‡’ÖY¬!,.^ãðC/0ƒPŸUÝ ™8ðû ˆd0bç:á4¥úÀgmÓ*®¸òÝ]|3¶Es¼ªæM795¬‡JÚi;Ê•é®m3hÈ V/{$˾c|Û¿gK]NÚNkNëòS5sº†þwêø¨€‹̲æû~Oä y^dFõ»L|ED/›Áüîn§ŽM²Ülèr¹¤ù|N'‹Ñé∎ šUº%E0dø¥/Ž`{ ‰ÿ¢Çǵ ЀÉ)ô0 M@rk{÷ÖŒV0Ðìij!¤- ¼gz)íÍ˧\ šÃã’öîOÍF¥·¯0!Í6ŽG¼¢qÃ=ûâÑ8¶×Å^õ8ŽbS`" †l°ÈL5ôq#Ô‘¢ú’OÏÈðvD?WÃ; `ùbMÐÈG¬Æ -ÖÐ7 qÐþò~îŽì3ߘ{‚±EfŠN„ßb g™÷’3»p.ãïv¼æô'Ž•ëªxØÞÏéžø¤Ø¢x…Á#p-T9MžIð Ú3™éŽp§…‡…zaXbüà")Dëm>Ó&^•‚Ý+äÇ4WBÅyæà³|"è«.ì•~šp¿-knnEǼ8ò”ˆ.‰è¥ë¨ {Ûa ŽPÏ@yn@’]àOA®Ë$¹.ËããÆÙ%%¶Å”a¯×5C\ ~F„›¾Ä¥Ö IEPו(d…ã#Í´¬è‘nµ ´(1`ðžLuËe•“}}5^³ëz";ñN·Î°I¡RÓßH”7-‚ü £îŒ MPŽ)î8yûo„·RUöý‹òNL…íeÐ6¬A-iJšÓfSÛ)4 tbö‚Äc± |KB#X7vÚ)eÞ9û¨}šíšè‡›¦óä 9H'[¢÷„è]fþ´YsäàƒîźFm›†6í› _]Ñb6£Ó(9>¦³ãZÌç‘Õ±þ Pa±} ’ Úwàï~"è´{\?£Ù“ÆC ®°‡ýœ¬ ‘מûLO!¬ѼŒŽ•‘j‚‡e}!þÝ£§òƒQœŽ!Æ°¡N¡n4ï„xûƒÑ)=0€€ ô˜G†Ý8Ž¤¢h`k.ÛJ£úaŒ¤ÀýÎ{Ùï'0¬cˆF ó†­ b¥výVÅ , [çFø,öï(,A)+hGs^f  9_TG`¹líÞÈ }2\ßgœIÃŒðùŒsÛÇ/΃0’Æ53* Ýdá}þ d?øܬ?\IP@w}¥¯½‚lcGÖ\‰yvìÅqÁå$çêA´”ý¢>„fÝ"M—ì8u èïzòÎÓÿu=쪙{Ú¬~ÞáŒð¶ÚØ2 àK7žV"ï'äÍe/8R×õrÈ;"iæîTê,iCn(y–dWØL6B¦Ø%T,®{²Ïû993Vžw‰v9'I ,IÞ—–ÒÐúÌ}S=†²ù Aœà þ…>› JXx©æ1lä¿É?µ¼vªOJáHù9»ë¢÷4ÜeО“4U째:;À¡£*ˆ½ ­éèÑì )²{$iÐï­ÔÔl´%àày•;m ¨µqmšy³Ù.¤9l©ÒØ£¢`û©$eú󨚹×A§|Ì3Ö–W¯¥~÷íÍúw~g½úö¢ïн{Ëç¦órƒ|è²¹Û:àfïP›T[d¦Ï‹,‹dG ²ƒÂÜ*çkjèÝó'ôèé%Ý=>¦—NÏ:FÉ|6vîK cˆƒAOL¬Ûµ˜—4^W‡‡ƒÑä冽;€z¸hkÜHF‡­Ý½ÖmÜ·áh¤¿aûq¬¼/Üé¡*9¡ÏPo`(‹T­psJÛ nï#ì )àtŒž`20³„6çR¶ÜÈ~ÏLWA0Çìb‰ýlÊjRhØæ‘1R¬ŽŽL¸&³q\‘èèbd ó9g}¨ Så:&¾!ž&0†”FóÉûÁx‚žÇsèûˆHù»'…Zé<ñN]+Ø& æà [?°£ytÍpÔ½ @ÅwÌl¨æûëÂÉ*U„ÉØס¹‘Eå`|Ðßáùm]¶õ¶;b½=|ÅzkV››åuQ4äMüº±SYÓ˜ åÏ÷!ñšKß³ضgêŠÜ¯ëúVÀ‘½VÂßù;g³Z­.nãa*­2¯§Ýè‰7m‡ÿ¨¤dh\ç¨Þ] F‰A2UÆMBsöýþ¼Ê.&C6⧌zý½ê<÷¨Êêï¼#ÃÝVFO÷Ôx6XÌ`A70ÞùB—ì2Û%ÿ=h”®Û%»ú'ËuÊËm¨ú%kÎÝQ¾@¹m²÷$*'¶±ºû /‚‰ÿJ›-.üµHݲŠ¯sZÖóŒ”ê‰,’¼¶èºeÙ|g±ø¿¾7Ÿý/ïUü/…¿Ù¡ÖÁÛ,OOO7£†ä ùØÊoüßÿ÷Õùjy¿êN¾„}0ø~)|Öï½ a~t´=ÚЛGOŸÒ÷Þ{—¾{ÿz÷ñã. GLˆL‹è…6EØÕ®»·RÞ÷SF÷"倗 $ ŒîQÃm7ü¬d¾„­ÕB<ì#Ö#÷O®TfŒ0žðgã) Gy¨¦Vz0ÕˆÄÁ"/;Ó§0~œû+‚ øL3øqœ±Lx¾±mÄ—¡Œ„ÓÓ…8Α ‹â4Ã1ÀäéŒÅcÝÀÁ¡öHí }$ Á¤µcaÁæ[4¦RšÖŒãŠo; L…ôìDx``]ðL:Sƒý˜€ÍÏÀpò0z_›bþŽî•ÎgF¬ŸuÒ¹Œm ³‡¿8©KOçQ¶I@ìPÄz8¨¦ Æ9£­ Ö¡^ëͺ³Õ·Û:Ž¹H¬ÿP¥îéM<Š\sÙÈðnˆÎkF ã¥ó[ßOLÅÑv†<¹lš[qÀíeŽP¯|?bf¹Æ¥7d“´€I›hK%Ï’d&Iµ)õK{X"”Ø$ùú©£~¯sêÍó(q£O·©Ï QµÔªªÍÉå'.F#PÔr?.Š j@PÒYѲñ´šR;¦ë0LJeìöæxù”6…©ò‹`M¿h² »#Ýï/aw…óò­RÝ_ŸœÆÀY €Êke#Õ¶ûšö<†Õäн} m»fæk¶uÝœWÕãsâÇ4§oŸÊâÎ=i>}WäÇÒ|vÎÕË-£Õœ(̽ .ªÃD‹ÊRà mÛæ%yðÖfùÿ|cµþöÉv(¯%^]]]Nª9ÈA¤!ú~›YÛœi{Ù U =ζ“*‚èm9O7ºÚ<¦w/Î;6ÉËwîÐéQÏ&±g)x0AnpÙžVCÎuòoÌlê½¹Ú70\Aãݾç‚RÁÑx7CŸ z„ÖŸzm3ä‰Òµ¤3¸Sh¬; W˜2 Bc(ãSuèÆ}6JÇŒ¤Ä›ácï=ttäd3°B¢Î¤Fžæ±pÇYQ é Á®ï=è0À8ÔIû->dcír5q38b¬šB›ŒõqP½Ú CØ%iö ö;²ODÒøTXIØeuLHPËy`¼«<¿ Vq ¤pêOê.d(2®M¥Œ‘apº5‡CqÑ©4„%!³"ÖØÆ·{6ÇÎÂ6â”RæQf`DCÃÆ\WÃw!¾)Œ¾=V½Í-Ò²EÄ+ ãEPUÉQ% Øl?U ÓÎkŸcÛÅ€(¼W‡²âF¶Í“f±¸ܵÀyÊÌåñ¶©á-8ò¬€$S'ДŒ–©d­S'KdÀƒ L’]äQ²÷D%ÓXK@ .lš›ò“rY* }ËÅ1ˆÁdc‘p\¬vá ‚÷ŽÁŽ©Ï³Ò‚À¶Š!’ï(0<#l¦ \®këö+©ò†k÷ÃNjØ5.Ž9qnÊ…æc]¯©Y…fç1¬æº 2”§ŽüÅv^±\^Šüþl>ûƒS©î¾T˧ψ¾xDò¹9ó½Ö×!°°ð´B…JuùFl³ê6ÿn[k–õæÛß^¯ç[õæíöP7­G‹Œ4Msñúë¯( ûArçXfü¨ÙñQ¿ž(Å΀ÕÏ »é㥽vøÙ&|tuEO®®:p¤ÍKr÷ô´;"8H8–ìÏ{sÜ[ß¿RnÜÝØ…};Õªù÷{Ü?Ã}^ QÌêíšÑðî·DÈAÁë„p/=ÔĆ*ƒ5‘yH;6œ|–”.œ+Ó¢á:?<ç„Ûj¶ÃIüжz‚]«jÓ=Q± HFš‚Q[;§kVû,ö¬Nó ;*!t誮f:4D#Æ 9Êyo@X v¨~”r²p•Ìì±»R_y¼5zDžY,%À©pº%ò•ú“~ðK3Mì{‘}ªX¿°—M¬ Ÿ¤‹Á3\—–p¯D„ï8ôÝH·C@ŠY­WݺYœÙ dÁÚƒ}î ×CI¤T€¸qfs4ƒ3>wmÌëÙzsÿèüüVp×GŽŽŽËuÜÙ·$]ì<$pEä£4lvå QÉ,‘|_©,JL ¹I;w|Ï“÷ü:@B2HÒF<2q]ºŸÓºDì$3C®‡WÓP\UPRæëèÝ(Ô[„¢ˆ<’«îëƒ)¨Ø¾tšÍ>v/ÙK„ÏÉs”ëZ˜üØWÓ†‰&M””GíÙ°€Ã*Œ­æî¸>jꦹxÐÔßün³ýöªiêîó=óýY}v¨™I¦ È®£}KÇ·køS¢ó§3>g®¾}‡èÎYÝ|ò„ès'†›æSæ–U"•)¦æ¡ÓŸB³Yåj³ˆlzò ^~ë[«å·Þ!zZ‹#ÃOY­V«ßþíß~þcrƒÜª,ˆÞ•OÞh†ßõo5öÜ\‰Ö»ÄïkÙ$ëU÷ïèâ‚îžÐÝ“Ó.7IÂ"(é™ †I‘™™Œ5$3¯ÌÌ°›ÌÌÅ|'DnL¸®2ôMÅžc ˜¡þcçJoü#!л¬Ï÷ à†·Sû&ÿá–ˆ$tŠ@óa§ãN¦QEÎ=61IÒçü§/ÉdD–Ž zÇÍÕST«ÆàFW›7!©mãFl2 uŽ£ž5~*œ¨¡±a|(µ#ƒjãd¬© öŽiŸvÕqµ2ô}_*ªËVabÄ÷$æ~‰@ ‚0©í s€ §Àº¬#Ú Ž’ôº'Ì-ã€N}ÚGGOÇ,w66/'{ùt챡ã"ÏDlî§5jX›‡Ò´ŠuŸ[¤Þ†:†—s4"Èc“&ŽKŸÈy^õîæ ê˽¯B‘éEþ¾(pÕÿ”«Š/>÷—ÿrMÿò_Ò•k#ÌÜžV³‘£üĈ-u\’(™ÊñAãƒö°Eòi7YJÆÍuëø<ƒ%™>n¼hßsn¤Ö÷UÁܓǴǸøG Á“Š…<é D‹ôœZû¾Ü®Ì Ù߶¤xMôOYQ‰›u L™*c¶Á¼Ü°<™÷ŽžEË<°þ ÊKÒNL§Ó$ûgVˆ=…ÀŠå.¥ëê¢nÞ|k»ùÖ÷ší›W"­ïóR3µ>ÐöH ÁûsY»Úß%D_0ów›æ褚Ý;mè³gDŸ[0½V UUµ;™¨ßœZ:ä¼ýÙ LÛ˺~çÍíò_ÿÛõú{DëÒó‡„ÞçI5>ÈArV¯{$Uµfâ3 ÞÍèATƒ¥ßn"hâ[MÞMÃŽ–’­ê-­..èáåS:;:¢;ÇÇtzrÒ±IfcR½øhZHÇÇP{~5Þó‰¢Â“ä¤ü$hX€"9ÜØé"¹Ÿ’0ÅS(übÃnBÞþ¡æw¶ €I§íæBèZ0ÁÍðjœ€2>T½£UBïNØ™5RììhH0`µÝÆŽbsÊÙØ’Pîø`üâ³ 1dÌ~dD1†kàç-á;0Ú ßÖrS(«(„^„'ùOè3Ô­Ý©‰¯}©½êXs>‡ø{X ý8òl‡µ€B[,€<(Ú búº4„1Á$ )o´nØeN$m‘ÜÕ©~*uSw9E6ÛMÇqP·ÐË݇Z Ÿç1È0Ãw1Ø\³x ÁÞðúÄÀo ŒfñùU¿ûôò­_ÿõ_¿ÜusŽ´§ÕüH=~Ê&iÿñ°i}”@Iɘ¡ ïn)÷–ƒrÓÓiJõzQdôBO€ Y’Â!,m˜WÜ3ûx1N8ËÇyD¥B}J$ ¿,(C;BbPñ*]žJZA¦¼S£ºîa˜ Ô"Í%ÏÞ¬gÍ?>išOˆ|ö˜«W+¢³¹4wHx6ì¦È™ƒ˜J¤ÈEEU{.´ö±FdyÕÔï=hêïü`»ýö»Ò<™ªç³þ^ì8ð3 ¡É×L­7m{»¶/™—K¢å£Ý_ý¿whöʉÈëG|æ˜éÕ…ðŸ 5ó·ÿcYIóàÑzóÆ¿Û¬ÿ¿·¤~´îªTÎË4 å­d?ÈAòbÉbÃoóœW ž«yX#xT¤Å ×;Õ[EO@“¼ç‰ÆÔQy±Zvÿ—t²XÐÙÑ1Ññ|Ñ險ë¦}fû[FÂ1žlv %Cy”‚È@ =}Cì¹À1Ebq?-ôÙ&e¿‡Y ²x ²FJì‘ ÇPÞæ =ÎX·’£G mQe‹†Ç~TNÍ^Í+‚ú˜6žJ‰yT87bßÇ>#×U› €¤Š Ì#NéO‡A,éœ äQƒ™K8~yDõ¿ìEôrq€î¬i(ãa3zc–Jý˜ÚZR)Yqîyåð'†6†ÔL…`·…lþzµÑ&Àáôqâ8î¬qú0`Ø T`ø¼0}™`Lô…ö5ÓO ÂáhÙl¨nãØnÐ!y+3T ¾#!’ô µõÓ(±|†Š : ¹“ã±™Ùˆƒ,ýz^1WÊ#~s¹8ÒE߆®ѽÛzðû•plå”̇˜ÑÛ6ˆJ”ö)úzé»Ò5¹,¼æ¦ò<&œÜ%%šªÊ0BL¡ÎCÝà}°ì…*ü¡ Gx¦²FRÒ)»‹Šj‘dm¬Ðü¹/\&·wç’X!7½JŒùTQý”ª÷.™,ˆ~!rçHè•Sá×ÛŸ âW*’—ªŽi–Ü^!ÍدÔ/žµCºóËiµ®›wïo·o¼]×ßyÐÔWÔÔ/bî¼V`^‘]‰oS6DëGDïó;óÿÞ‰ÈÝc¡WŽ‰>q$ü23¿üH¶?XÖ¿÷ÃzûÇR/sò'"ÛÛ:þ 9È‹%MÍç•Ðc"ù´Ø ™`E˜­` C2ÀÁÐSAãºÛéÇ6‚•¹­k:oÿ-—t<ŸÓÉâˆÎZFÉÑ/¶¦ecuJ n DP‡A0Ø+#ó•ƒÁPÞA±£¾F ÷D¦„„çÆt&ãΡÅÆk±'Y‚Εf¨tàIÇg‡º€!æ*¡w 2#Ê*SœA;Z¦/úäð‡7â{fŸÕ<˜$ Þ@¨‹3ÃkÚ¤¡tz¦H|wvÍyÕåC´vëé1 mÖ¦bΛó#‚”Û6<Òús,€©YÍè½Rß\ ùô§?½~ðàÁå³fŒ¨Þb;À ”ÜV"×}T÷ d_»èî¥pœ©Sp^4`Då:þ]”W .¦ÄãÅ–"Tbׄ V9N ^Œ)l•ö|YXÕßG›GõÛ‘ têž}ì’}²ëÄ”–³­ªÕZdõ´â‡ç oÁt:c¾s*Õ«'B¯4ÍgæÌ—%T IDAT¯ŠÈ‚¾«{Ü*XÈmÇðò‰šM#÷ï×›o¿Ùl¿÷TèÑ%5›E©èÅ|éõñçÛªZ_0?¸¬øa%òÝ…Ðñ\êÓ¬–«'u}¹ %³á˜yÙ4Í9ÈA2’z~÷òˆ–ï ÑÏLâ Œ‰ÒÁ‘ÁaCIYMȉ5á{Jß‹îµÄÝñ¿«í–.–KZÌgtº8¢;''X²˜ÏC"YLÝ¡´sø Ó d–¥Úfš?î••‹áÞe©lLöH…–©œ¹Ï&›7<ÇO ,“¤[™W=Ÿœô=7F¤>=ôf‘IìW|6Ïf† º^>å&wNi–ΙÐVc•8ö%ê¹B£ï0Äf4N ²¨}i6)§¹çýÅ##[(ŽA¥½býýŒ¨5 Ä ñ>$¯piÜ Ã¤¤ãy‹ŸSꄸF„÷\×!·Q=mÈ¥wû1$–ÛŸ¼‡ÆŒ§OâáÄœ]„ó"²„òØ Ïª¥ ŸÙl·ƒ®ÇFÁ·,´¯<†œ!¡®¹yF†þõ÷E¯òñÉ¡`>¾æû L•ªÍ6ùxK18ò÷þÞß[ýʯüÊ$…ýG-h¼)P‚‰\?ˆñ´ë”š}ÇlfÙu Í®pœ]L“$Ù—£ˆ|7C˜®˜™ ŒNØLa5ü¥@iÀ—=Xº•”LM,)IæU)ÑK'ú¨DgÝ%7J¦BÉ u‘-󦩪ö­'O›æYÅóYÅ'gD¯Þmä‹ÇMócñݪͿÇý0™ÒQÁ†ÀÔÔÒ‚ÁªGb<¨-U€ z9úð rŠ‹™àª1êìû¥ÄAÐ#³@¼cõbWèQ~þ®ëP®“ì·tMÓ2ö˜WK¢Ç™¿óýùìŸ}VýoæÕ?Z2}³zØævUïÛz,ËËzó{ßZ®þÉïl7ÿêMs_‘}ó÷YF¦BNT”âˆÀÈ®<$SR û›: '¯-û€”©ðÀ Bãïíþ±ÝnŸC%¶‹²ïÞçQPá)ø…ÇÅU=Û,0Ó÷‚~1Ž¡²[Å×aÝÏïMOG…užú=3F¦~'o&¨(æû¦d{DËT&ÃÔõ×É¿#}_Ô«ÙìÉRäüábñÆb»=»SÍ>qGäÇŽù½´mäü¾l÷÷¶ëß»éòX4C5Ú3ŸÖÈ M®*ˆ±+Ñ>Päº`Geõ¾pS /Ä¿w}?H›èq†ä 9HA¤‘w¸’­ÍÍdFê<©÷4zvÉô>úýÑ„K§L°zúíf ßkŠ>°4‡Ýί®èbµ¢÷ΟнãzéìŒÎNOi^Íú;0-È T÷*cÒÙ˜|uÔžà!>$ †ó„’WÁŸh ÷G¡VkŠ45¶sNô¤‘ïÁ‹%‚@"¸ B½¼šš72ÜH—`@—YºÈðEëÑ $äTùI ìs¬é¬Ð'NƒÉ}ãÄ0&ˆl,ñóð>ÄN"HTÌö±IçOèß4%óx1ä°ñ±p/°G"Ä{Ç@½¤pšŒØ„µ1 NÇÐ2Iç³HC#uc¹a8Ž·ŽW8rY©4û¬´¾'Î õ„÷.ÿN¥¹…·y”Vë•éó)/@è†RèÝØR¢ð.¥æõÁDJƒŒ’n¡"†œÁdC9ÛŠC)Ü&Œ…1²;EšºnÞ¼:>¾5vòµQf~XÂ[ŸÉùIZå}ɵޯì£È£LJS2N¦d×µ%†É‹r3Åx @ÅÒþÏØXŠ²Î°˜Â Àö¥—k¯»)jaW ›ÔìHhvº!z\S u.±:2è‘ QFò½û$%ÉÏ›ú&æ|‰Ub-TUÕÂMõj>?¿jšó÷ªê»'½Üð+ïÖ«Õ{"Wd]W4âKò<Îó¦p ;vìê‡ÜWùóüìRß¾rsy×p¥—‹ ;ÈAò±–ZäÎûfÆ®[fUýñèZ/Ô{Ù˜G“‚T}2lÝUŸô;|‡¶“¦0«ÌóiëZÿýPÇmÓÐç—ôèêiwÒÍËggtïô´Ó3UÇpó“Ñ)8}Æy;F¹V†çÎSiè Ug¨F.$ýô/ÍÀ.‡5S´œ –ÉŸ°àcéÆ›Áž°™tÒ]4ƒêkD®èÁ<ãWr4é¥ÀÖæEoìE‡˜‡¾D›Ó(ÿ© ¬Þô¤wåßc ¼®áúQ9ÑšËÀr ó¾·ç"þ@1Ìw“ùÄ“·ãÙZd¬ ‰*çý‰µyî%É0Ö˜cÉ@PtZ}}ì"Ң벸zZù©/á~÷(N·§€0:UËc—ÛmÀŠ¾Ë‰Fˆã!4uѾÐkãPº½#˜¨-ä ¨r‡˜ñ¹€G€ëšç©x€‘«„Ú8¡‰ÃÚ¨cM2«è]Úm>ßH® Ž¬×ë§'''-*sv{ÿèÙ$í¿š›$Ó̧¼çS×frWØ©{§@•1É”DôVXM±ÉöN ü¢ lXuÉ=8¦€De „ÑT ~bMw^?§Åé’ê­¼²z|ü3Oß¼÷ïž~çòa›° à.°g×gûØ!û˜!¹Œ}×ãÛÚUšãTx7 žrÕ4—W,—U”6•c—<‹,’ÒZO¢)†'€¡)0cë¤t].ÿº –U*ïk¡‘årùäë_ÿú9rƒ¤,•=¥»mÈÍ|Þ5ˆw¨‰ I­h7ã‚÷B?ÈŠr°ÅuƒF’òœ©QŠ0,Ñ夯D#ŽˆãÃ^(Œm6Ì —ˆ¥†ì |èÿ˜L5Ü­‹KÆm$)Fú~d¤ü5lÉLñ¸\²°î’Hê³ÜÓÞsÐót.å62ÈL ƒOó=蜆z%|r<*Î9|n`X3ÁøÐÇd´¬WEesXãØâÚ Œ œ8Ю yÄál ‘ *`g˜Ö_ësSºˆUH¢Z²]Fajã ìy#P ýbˆÎz³î#ȨC,PY]?'MÂJsRÇH<™N=­RÌ=¢Ïô¯Øg„I|A ,ÔÜ=žƒ©KÒ’oûL§¦Þ.Dî—;ðýÉMÂjZÖÈÕóΙ’V©o“·V7ÌM²ËÐ+S ‘m}ÊÀÜ•ç¡tý‹$ûÂEZCŒ©Ù0Avúa qo°Èô‚Y­õ‹ãµó†æwjºÛ‚"w/¨šomc¤žy>¿s¼˜ÿÄñéñºû™Wž¬—«®.ß|úæù;«·—íKMu$ƒ•Pà]ÓpËÌ×™§ëš‰z¯>–ÃjJßçß÷+× ­É÷?+ ¾ã-0²Nz¡  i*ìŽ&ú/ÿ½‹áQbš\—‰6Œ\‡å&"O¯ÕY9ÈA>–"Ç|ÎB„è9t†€ ŠÖH6·P÷ëf¬A£'@‚RFtãVCÜ<—v}–®p4$”u"B«%/¯èä|ÞüÒÙ:=>¦9®Õ)ÌÁ Fç» ;"SßÉ}é°×»ýÊLc#¬ä:G0@õ!F úytÓ„‘7z„‡.P¢åW06ä¾åÀF@À$‚%‡ÄÏ•,…RͲ`‹:¦H“Àž‚QïßÁu:—†ôMÒü*z‘â ˆƒÀ&òþ4G!9–bp“•I:àE„ùhU¸ÇçddHHš2á¸}:N¨¨ÖjÛ Otð.ÒÞˆ!PL 1ö"êÀqÀ‘·âõM ¢ÑÓq§Xn% hÍ`NÈÁÏ5Œ¦KêŸÖ*{w…ƒ­#¯ˆ_ÅFÀ‡ûðØfW|mÁ÷W,'L|?€gxš²W47³°aÏöN+±âÚ:6}yójö 0KÞ·Ü„9²‘«ëÐõŸgÁ°e‘”Ânr8 íI¸ˆ÷MÉû5âÒRø;±i2¬FN-Ì4Cû©‹ ÙÈé>š ÍO·töɧtòò%Í+Û lìªZÌ^9Yœ½rrçìgÏ>óÒÃíÕê­åýË7¯~ðôþöÑæRêz+iJt¹}²4¢¼¨Üà=ÞŠìÿ(Ó¥Ž|ý®2®ó^L ?JQP¤TÏÜÆÜ× ¨°†\',©ÔRw,¯4îSeQ?77³ÙìÀ9ÈA2)²9½¬ËGú}4Á0žÝ’Þè+›¹ªØGo²zÙ 3†rÍ^cc‹¸'œ¡.øP/›aϵˉhµÝÒêâ‚]^ö Éé)œÒÑ|ÞŸ¨S¹NbÜ‹@ÉÇǹqk@H®‰=Éõ&ä$…ï‚î”,É{dÆyòؤD¬Áè'¦W`ÀC\´­È"qàgV€WÐÙäC:r‹ƒ1Ø8^L9TP10’ú Æ0þ—ÀÐî¼óL–¸Wq µ°AŸž:îjjV„äi ¬bì5¢ü†âØ©î"£lüt4$:µ'`¢8ø‡ó‹É=\qÀ0„íhÿÀu¾ 8›Ç¿ˆ‡!w tyiøøÛf ¿³Ú6÷uþˆwµå(Is¨nú“»“Yƒ#q.¥_‘áä/þŠý Ž æ¼È:–4Á)S•)­'p̯2FpÎ⎠S­xN?æÈùùùú¥—^zÌÌ_¸Í <Ë¢yIÚÊ&ÙÇ_’)†ÈÛcÊ Q)4Wáô²´2k‰b eF1€ÖLTsä™r¸+¿"šlèäÕ¼Ò†Ð,cˆ (.º˜âIöÌÎmUÍæ¯Íó׎ïýÌÝ/lß[Ÿ/¸ºÿô僫ÍåæR–²Þ®¶(É`ÉMŽ÷sÊuB*h`7eÀßĈ/É.†Õ³ ¶GÐf_?^·M»@¤ëôéH3•ÓdŠåVNÖÈúäääÖŽX;ÈAòâÉîž~êÇùí‘ãX%„r8(Â4²!À¨cj™ê©Äd‡„Þõôà¬xeàoo€¿eÄVEÝ ¯×“ö¤›å’Žæçt÷ä„ÎŽOèd± EvÓ²˜»»káÔ}0²Ð«mF'L°²¬í”À3–à´Ä†°¢Ðxd †Q?À€†“H8€¸€Ó2˜¾|:€>€ÅÐ ’ÁíÖjÿëõ©­26”-I0±i2ìpLÒi#’ÀÀ¾"èU¿0D‡Í€-œ÷š!0¹(!cN”‰ÞTŸ?lã3 ±yˆè Œ•ú®…r0¼oqÌõ¹‘± voÔ×Sí|°ºŒýЗ¨GCŸã©Dã8­°Þn ׈͟T·à<%@‘mÇG¦Ñ«$%d6èTs/[ W9›¬þŽ’†#bè ŒsJã˜A¸Þö}¦ùOX alqmgh_CryDt«©?® Ž´ÉôþÊ_ù+oóáÏ‹¨Âo±«ƒ3•ŸdÊ«R µ)Ÿ%#êEÌ#ò~¥Ä–hú=¤é³áè+†-°&ƒ§[Xh~VÓÉËK:yùŠgWÄU³£êÁs2!¶Ö™Wa1?Y|f~²øôÙkw–›Uýps±zoóðêÝõ£ÕƒõùòÑæÉvEÛðÙn\—eR’›°vÍtLnÜ{Vß…f82MÙ"»ÞÛR_ÞVö•µkÌ3H²¯¬©9ÄÌWóùüÖŽX;ÈAòâÉßãëËÿêÇ¿ö=£c“jÀ¥O©Ï aŠÆàg–˜0˜lfXÆkªû|:VØ M¨8®§ç5qª5Z6Éòüœæ—Oéx>§ãã#:;:êN½9š·¹ðUÀŠ^ã$¼z ê‚Æ‹[#C}dŒ€Õ#(°h2F¼ØGÃÁ ïñ 'h»¼ÿ#­¡d ;ðAÁažŽXCõ1+ ³ÎÊèš …~GC.-!o„±Œ%!ã¼"®¾eŒM€û \„¿Lÿ e0Î% ë* «ÐÒ«ÅUgœ# qø/VÉfp0´aÌF›¡ 4&]Þç0‘æTz.‚.Ê,÷oµ0»ÆCóSx „ù8#Ÿ“©W†9ToÛšmljzWœÊ˜k›¨!‡8êþ¸¬ÁùŽ¦ ]Xc3ô„b;mH Ä„À9?)Ï¥vámËãÑ­²“¯ Žôuá'å×ñã!rÓjø´¹I¦€’ëÄå_×»[ºf—Aúq| ¹!‹€ã¼Ò!Í–ü¥µ«†“gDÇŸ¨éîç.èø¥UwM5«}ŸuóÒX¾ø¹ødhslºo«ÙéÑéìtqºø´|âΪÙÖ›‹õ£ÍùòÁúÉò‡Wo=}·~¼éŒÌíô†¡27•©±<÷¦t–ÂÅ®;gŸ'@0ƒ"*´þSyDvõý®õˆ&Ƥtý¾ïJå ðËÌwîÜ9œTsƒdŸ¼)$5s»«G,.0FKžÚ¸">ÆηTÐð]2÷’«†¼#°³7è¬(å 0*T‘·gºvBf´†Î度§ëÏæt4ŸÑñ|AwNŽ;VÉñba¡$1ÇYá4 D‰0Qð&…Ó`.ü™õ§ˆŒ{MO8I˜Œ•Â¹¯Õ;l¤<Á0û¸Û IáBcÃLW¢08ŠºbDÍæÁ õA4• À¥ˆ0Ì·Áq.±Ath§BÈcß 6z(°?|Ò2mæ{e\ÇV¢ƒ:2)¼r]› œðö!b†÷Rì7Ö÷:Wuh'¥êc—P:„9éØÂÄ4ìÏ0.èü°e 3L(²½¶ ¡Ùn©ÞÖ}2êìj ªÔŸÈ}‹³ßm)ÄŠÆpÔø40fgŒØ;)ÞoÉ8"94XøMd‹…ø/dù1½sqtt«¸#³ÙìAÝ"7¼ïE”SßQ›¦ŽÞ•Ü°ôwfž\·¼ë©/ªØ‚Ð2ùG÷u È¼öòÂâÒ)Bs¢Ù™ÐK_ ºóÉ =˜"´6bÛŒ%ãG0¢ ·Ñ #óŒg|Öþ[}²ùÄéf³ÚþluöøwÎ÷ÝoʶÏK‚2•£dW"Ö]óqJ²!||$¥yz]ÀãyFôhÞÌ»©\'ñi‰É1†ìún°QŽŸ*«T?(çñïþîïÖ7ä +iD¾ÃÌ[éÀ‘`+äf•J ìw ›æ$ =CÂËÅÄf×±mûL`ë‰z?>›aßU£ ]1›à]ÏSª306ÛMG•ºZÓ“«§´˜ÍºÐ›—NϺD®³jF© Ãïà IƘ†ÆðÈXËîœsàŸ9ÇC 0q0 BcƒU&aïäQ[8ØDˆ=p`äÐèóQ‡£QòÐh l }vHÌ-R2 -”Fõ1 4ÔedX&–†ÍÆqÔÜvˆù#è¤Æx3ðèÙ:Ž§Â@?ÛÁ¨t@@$â(ù¸g˜ê†|Lxì`œb)M¾Bðž+ h™—¡ À†ÂÐ2zÌéÀ0‚±-^ ['’Žó™àœˆõ´9!>°m¶M¶Ú4ï#€ZÐ10³tÀ’ÅÇK{šu*šFð,±ESçRœnÍñ$ýÚ _Ó>ÍPJ¿öð›÷~ðæ­ãK79š¦yÒn\"ò±GPÐz?Çgc3{d§Œ¢›š× ›¸©ñú, R×l›Î(&` ¸ø·ìâ.ѽ/Ý}éøNë\j¯Ûúæ¡ ©R ‹ßh3#_)pQ ´L\¬yŒ´7RWuÝ3ËŒgrÊÌŒq®Oþ=!8¹)Ûäƒþ×É›ñ<‚xÍp M)9éu˜4¥°¤©ºR>]eæzîûl3í&ù]°/êº~ø/þÅ¿Ø~˜Ì¦ƒä Ͽ̅ߦ¥SpGçT+š ž[Äÿôª¦²:\zCWµ†¢ñfuRöxÜl1¬Û`Nš‰Qî F¦•áùbë¥Ð¶iÿ5ݱÀ.Úcéå;wº„®m~’h_4f@ƒîÒµ­êâ9AXMÐÊ/÷Œ£a9ÖÔ»+êÎåxÍd cH“Z˜[ƒÅ¡nÉa`10"ãdµÙ+ný@Þïªs±$€¶ÇŒ¿hК GÕ–ò– “bS€Æ÷e†³µ'%  É`ü/)‚Œ÷¥þMešf;”™“×Æ7N¯£Y´.¨ Ç[ë¸xì0“ÁߪðÂ'€(Ž7ö)~6Ò§Ãg–µØ^9$åÙ@Ãß/ÀÎ'¨BÇiç­Ñ)Ÿç݈ŃC#€Ùø‚ yC¾Fˆ2Lô´*Mºu£ ì3Ê+ÁÑjiÑÐ2³eÂ{cPÅ6[‡§{ã.D£ûÙmJÒs!Ùl®Ç>vHéú]s­½¾»ûrŠ” ö)°ççuŒíg=ôO¡™êÇ©ºaŸ–€*¬%°¢¢æú”Ö‡¼¦”걯žùþ,íÑÚMÓ\0çë9ÈA’ŒÙÙH:ÒƒÀ‘ò{Im¥/]Á IDAT z‡D©¤–aO:W`Bžfð#H#~¬e£I ‰IÓ‚(Òƒ-îiïED?ŸaxŽRÒ“-Ö%O–Wt¾Zv!7÷N{6ÉIÇ&©¢±`ƒŒj±_ƒ?Õ ü j©`òNíÚ²óŶ0é“‚C„@0ÃœI¨ÏH0Jñ„A Ÿˆs†ÌkÃ/ú6UÌ-ú˜ãfìxŠIk#vYÙ,zŠÉ9“!ŽIYãC²ã®œÀ×›àýÙÁ€Â Áöj=‘•ã |nÇ/³åóÑúÀIIÏ,£ÎìÀùÞŠ èÞ4 cD þ*³`2ÛFá4¾ P´PtÊ2xŒ`ºÆL"­oËiÿ™Ý0(¼ó¾zN@ÌÀR‰ƒ+}AÞøÚk¢³ò¬Ð…Âûs)1ôŒ 80²*Hb-Väó[‡úôkÊV„Þ¹móFàHUUOÛã|‰èô6+ñ¢Š,Ê&i M=ñf ô@™26K,]yöýMÃîŘݙôµVâöëírC~ð˜gs::;¦“»'ttzb}b #nBis¯OL*`‹oF’Û…ÉÂ3|cêT°ö€bØ°Ê`M)¼Fe€’ç”#Yö}¶/ì:aaSß_—ñQHwlÚDRå©úî ?ÙÉîÊl‘㣒PZòç¥úì»îº}ß‚zMÓ<š¼à 9ÈAt½Xmò1="‘Wòzĸü‚rOnðž¥`‹RÜv™íôÍ5"6c„“C ¼ºu¸Ñý_Ï•©ÖŠ`=(±ÓGü«Z‘é íRìPÖÕfM«ÍšÞ}rNg-›äô”^ºs§KâŠAxä#pµ˜˜t‘ƒ¾DÉkë`Ð8é"Ú*˜GÄëaîuû=ê0Xñ˜£A?GÏ»^.Aßóï%µ¼Øx ‚=ä!Vñ¹Ð,uˆ Aã Ýtk¬ËгúM¥àCC:¶0Ž •À(ïËÑ¢àEìã`·ÞèÃ+á:¼›¦9qúú¸n @e˜°‡ãÑ]þr›q\ÐKÆ°9ÿd”ß` Kø[,3|"»¥Ì5ÔYûÒÙI†e¡n­ïSÇ}[×´^¯»#ÎÀI tx†3µ°€ _w¼¯ûwRHóΰ2×D“Éö •¦]åÞýKÁ: ï–±ã¬L ï‹s¡Ñ¹0´²Ž¯&«©¯5-ÝÀ!,u#ߧ[–#o¿ýöêSŸúÔáHÆ÷!j´´žfIhG~,S^ä|ßT¹*×ýìydŽ ke#ÜáàÑãGE…²•íÕºû÷ô½sš.èôÞ)Ü»CGGGýªa†"±qÑ2ß< ¼è9°zCx†@â6ý6„ð à ‚$SÀH·i†¢»òì2ÚK „©°]áZ»Â3v…}ìº÷Ã’F3jN½§4 íbèì;KÏzÖT=ˆ¦"û@›]Ò2GÚ+?´Á8ÈAòÂÈzEócz¤GEqÿV¥†Ä§¶ßKT¤ÝS ¶8nýÁ¸’°ZxG2«à£¦Šy¼Ñ Ô÷‹Õ’.WKz÷âœ^9;£Wîܥ㣣~Ý7 ÊE# td@¨Î¢@Èðy `¸Ã/²Ñ±,§­%)‰†/Ò,tÆj ètÖ)cpÅ80t&e} ~φèÈÈUï ©)êS c2̳~ Ñ£ˆÆŒðêAŽì<‡Ó>ì0L"€b ŠÍ ‹ hK ãkŒiBrÁ ƒíáþ7˜$lA™8Þx¥¾üú,vc>Î#µûÒŠ'=Îs¯·¡+Q,Yjó ­ÖлbÊ4 /àøõ0˜þ=êŸÝt@Gå=Ò×pxçyLZêÆ¡Ây>¬{Îâ`‡Bß’?ÐáÖ–FH#htUµñÖ’FÁA —ìOÝ΄Þ.æ#¯¼òJ[·óÛ®ÄÇI2HRJàŠr]à„vä*Ø–ÜDžuÀ„ûÕ+æá>5¦Ðé GYÁ[½}º¡ó§º|÷œŽïžÒÉKgt|vB³Ål´ †Å—âæEÉs`ô¸ª÷jµ(qS7°éûÑwöY“ðÿR–zšØœµJéó]L•}†q)ŒïÝìb…ìT>jÉ WwõEéóÜÏ»úhê¾}}rõb_èÌTÈÍ”ì)oµX,€úAr½2oŽ6B«û„vmÁãLÃw’bdVÈ Ýg{v6†«jDôkö|Ž)TDKdI‰\%˜T}XjPFлî^ÿ>Œâïp‚; .Õ°ßÛ_W› ½ýä ½wqA÷NNé•!7É|6ëœâ ˆé>ÙiƒÏA€Tc‚DžP¡¡ÀŸ"j!ÁHtv@tpYȆy¤ûç8•Øš­™123!ôƒÛÒDH< ÏF“A÷„ë Ø:dmŒb} @ˆ†DÝ/¶¿ªœ¢LÆ“2ñA 1RB“gZ7D£ŸÿEp±ºaÞ½ÄT\œÌ•ƒ˜ÖGXWºÐ(œkÈîIì!›Sy uM±ö$ŽDÆ¢tâ+¬"y|¡ÉK°|´'Ь·}ÒU,SÀ#÷@žGãy;È¥ åEãÁã Ь‘î9Îo4üÖl†wŸ{Vˆµ¤Ïu ËZc ‚œ(Œð£¾/žÝ:;ùFÇoÿöo·Gå<¼íJ|EÃm–Ëe÷ïêêª{!Ú×ñ”—hü˜O€X²/gÀ¾ïvÕçY“6ÍA%xx—*"ˆp\„ÇÈ».Q ]=¾¤‡ß{—ÞûîÛôøG´º\Òv³í²àKãôF,&ä)áø g•·etŒ‘Õ%»^cìÚ뚦iÝh¬a5ð\Áëó8îBvåÿ˜Êu‘ç߾ܥб,û…hn‘>Sø¬È€Ä`0jTjÿ®w:ß»‘S«ü¬]ïö.ð¦ÔF}3¯êº>€#9ÈAöÊó¯ÿá%7òÝÞæö­aïª,Åv"M;(ãH·×½÷@5õ³¦1CÁl*dc8Á†c€æ:aÓhÈIâUrÕ^=ŸÆÀØÑ«ïã‚OÉ`éª5ýmׇO/éû÷é·ß¦·>¤‹«+Z'Ã+·#L†@Œ9ø€m³r eV/e§ÙýúgfÚXXwA]N q@™ØUÎÔŸÓÆŽl>@”ñcÿï‹s¼ò7ægj Œº k™pÕ‡Ø lÍT±þdkCr±˜Õ·ŽÜøÔy<å•>Èû“r“´á6Ê*¡kzòKá4*7ñ¼ï2†Ÿ'ò õ’Ñ`üNüclpØ ÜÛÑç&Ù\=¦§ósZœÑñcZœÓühN³ùœªjXZ ÌÒÞôñÈ`¬²íÞ¥Žæ–aÒfOåLËSÉá2áîÂg»B>¦®ÙÅ&(¥ßógûXM7 ùmÑ÷rˆ1UßqSÆÇs$—?Þ4u_éï`UjÇÔïóiUUÕ­ž?ƒäÅ•j6ûíhàÒ6cQÀó À‰zí؃¶h¿QJ–Ú{ŽS2Vr/o3\ïñóf"›eÌÞC j†–bÓ=Û˜-ÂÜË+v¿šiÞtEC;£ò²WZ¯hñä wŒ’³“:šÏ]Óc9zUd`TQ=BÆ‹÷‡1›lÁ‘µèf$ŒßÞ|5À\ªF†Uç¸JDÍb6õšÃÉ !Ñ+Á‰ø2l(õ)¸’¯á2!× xâ-q+Xi $;4ã,TÈAŒV°D³p¨:2$ul`Ôx«(†“€ªâÀ$õl?ÛHMŸ>¥'ËÿŸ½7k±mIÒÄÌ}í)vDœñž;O9UUÖ­ª¬VKÐ …ÔB-QÐô z•h„ „Ò/Ѓô¦§ª—nD‚ê¡uƒÔM•ÔÔ¤ªÊ̮̼әOÌ{ZîÂ×r3ûÌ×Ú;vÄ9'ï¹÷nãÆ={¯Á—»/w³Ï>3›Ñ¨ªh4¦ƒ½=šŽ[ ÄWžOÌtw¬îa«“ [02JˆŠÔ8. xd&¹N43AA 5dX09ຠˆÒo΄>˜yOá`îX¤ÂXÜvLLÑmòzhoÅ¢ôªñ¶uïÇÁ÷±ïo£–tçañ,e( ýr ž¿=yƒer¨Å€cQKÞ´Ãä$‘ûB¹î4)YÝÚg6inK)»*C> †"úæuúƒš0ú©W= ïªëxºtA~€rn(èa_P5§ ñ“ï˜%+f,óKr‰†5á÷¸ö €`¯ÔÎf¼0Y5(XÞguUÍú{êúrep$„ðÄ9·G¶ms}l H#&œŸ$I“´8ö2oöºû|ŹŸÂ¯Ž*8]°X!å&b>Pä^½ÄKBûwÊ,}ZÓâlA~X5,’’Œ¦£æ_^RB¤ìµusÁEÎfÝ”þèÂþ}@ȶ9GÖåY'ë ø¾kÑ È«(˜[¤/ìg[ a¬ëãMÀÉ&¹J¨Ü&¦Ç6`ÎU¥ªª”Œµ~E‡z';ÙÉ+&ѹŸzçç‘hX&÷ÃÝË¡5Ä{aþÞüîdËnÄhm~ë©WEx†z~o±åÌh!5$'³L”¦¯Uk¸­b”‹QÃ6T‘¨¶'‰¢a7@ˆÑ|µjò“]\ÐÞhØ$“ M†Ã¶`T1º .˜gƒ®ˆÐplT®hN 1qìE—ÛG2ç²×—UÅä()˜:ÄN{HÁ´’Ø0 ÌWœ{Ø~2¿2m; (À‡iôxgïbŒ`„o!nÁ O)`¨ F× .•kÆq_hŠ‰&Ê«÷f•±1”ÈL ¹§N˜v½ Z*S¾ÜæUè)µœ'4½®íL6@b‰,S8}S‰¦`†%ŸMÃàÃŽŽ/ál æB:ßØ[$y†t4mðÙjÁãð7…X$)Æ^fY‹qÖ—)ðÂQÀ®æ*6ÁÅÏn?}úÂpWGRBÖÄùº†Ö\‡Mñ<†!RÝ“ÑFÙxãD®f'\ñ¾Û&_F0%•uJÙ‹Sô ‘çå{Ý …†¥±ƒhàZA]¨Ç]ÆÖ‹Uó³8Ÿ“?ªhz{Ÿönìc+Á„¾“EƒÛ"9¹]ƒ‹„fŒyæç}7…Ëô›¥¼È9ô²™L!Wîzã)C®±lÖLÛ^ãE²?ž7·KáéïüÎïü|Äìd';ùÒJtßU´âO e£0[¤ˆ1@H¼¥`£ÇbÏÆÝ36Uh¼(ø¾cÜ¢Øiu |, ±JØB.&ÇE`d85}õÁ Ábˆ:McÙw ž8mOþ<k§³9Ït|~NÃAEûãq–¤òÀƒjÐ$õ”&@«žÅ‘cïmÙ#Â(àg¶€õX|Vª7`Fp“Íbˆ0øÒY…ëHï` [s±3¦z…# £zÂv°OÊ0‚K—áE|àçDC>ŸË³ë˜xoŠûØö™·É|Âý„ìjtÔaþO¢aœ8*”íÎMH¯æ·æž)Cü}´¡>rN1ƽbæ‚°æ@HJï{)~ßäÔ„«TŒ¥ÌE¸—[ú¸vü£N }‡ ¿J$ý½¦Ò‡aàDYBÌøA`Åñ:ù‰tRÖ=_ÞÙ$¯¸*šê༘«€iYu‹£F°Ùä=îgû·{¹–fsM¹28²Z­N‡ÃaŠ!ß{¡-ù’È‹0ήbäõå†` $MÒ”Ä'¡üÌ&ÙD»ß6 bà:Ïðó”¤XÄÅi¤xË8.J§@ÞïB_…±!{¸­,ã2sƒsWpyLÞêùHk[–îÿbþ±òïm>2m ’• 0¦Ä€Ä¬8k›G\hU@D=ݤ¶ $™­–4[.è|>§ª:£É`H{£ÝœNiosÜ `3É/BdÌöÑrE$DðÊô§eI²kEäHHìàY%qµ:ŽëHñ3çJjÚóË•Á‘{÷î]_„n=ÏÝ¿œù,Ö˶€CŸ¹IÖvpèÍ '×ZGÿïËYÒwíms%¼ªâ©žMèøGÑý$¸ÑÍeœ¾·pãoÖäï8WlµÞ¨ »&³bbCóf#4¿tÆ…;¿ÕÑ܇ô:ó¦‚ )DÑ2¶J@Ä&/Óß×å¤è3à71GÊsJã|[àã²x¯—Í!£ÙtÏuàÅ6@­a• ôõ#˺÷y]ÈOß{Z~¾i>ÿíázl·^h6AâV»Ú£CÂ9Ìïh I6˜ŒÍâÀàŽ­‘(9r+ ˆ ðÇÙÐ)y|6Üm舀¾(" ݤ`†'ßÍ)Ô“ˆÕÁ³°_j[@5× |£ÌŒì?vW€< “¼®i¾X´N8<@Déo†ßÀhûJß!ž ^Þk|$°@@à±´¼®0¸ÙHˆ2¿¾“þðb( £á^Z1‰ƒ+ÅÖ Üvyè5 Ë÷ ’T·í361EÐz repäþýûq4>/•ÿªÌ‰uÜuå26ÅeL‹¾ã®ÂÄè£ò—†$é'ýNkŒàë°@6*¯Ø•DçÂÒÓìQEóÇ#Wýé*Nî­ÂôÃUïÇèo¦´øv±væ'‹ï˵˅š½ ü?¤Ûͳ\¡ ”ÂbbW[|Ö'?<†Ò±I6å}²MXFKä*s¨Ïï{gúX/Š½Ååyñ~}ì˜uí-?ßèÄgÙÄ0Y7ëÖ¢MÏz•ï^0’æåÒ9÷ÂK¬íd';ùêÊÞ`]<ò¸·¹GvÔÈßCž.ödÆž2ö²ºETðÉ7Qع¬‘Ø º¨Q@Ü좚M>¾1r¡ðcz6ÄÚûûØ– v`88i›&ŠÜI׬£–†–ÝàI h:HsOt˜Æê6‘kú™éÖtŸnîï7l’ÖAÇŒ¼ö/Iõ Ô[Ê>* y6°Ömî‹gÒr2flL¢×¢zZ¹6|ýÿ:ênD¡¨ˆôžQæ•€¢Z£Ó†²à5Ðáe¿·,L @É8Ò&ó0 ¼bÎ }tÃÐá1"˜Ìå8GÆÆa©ìh!øÜŒ%2Éì³jrQ«cÛ¤©ù˜"q+‚Wl 8<ÞêKÔm‘¤GÍ— )²A¢ó pa;(xÙJ(߇Ÿ %vÎef–²D`ís³NË:QàcæŠø¹Y«m÷ ó£e‹YàË®¢ .^ Ýà§ÛÈÕäÊVÆïþîï.2Uº‘çQ¤·5Ô¶9w“·÷2Šû6ÂUó)\Õ€»ªaIŠvm¢Ÿ%Íf³&ä_L* Àòº}FXŸôyÙ_e&I µ©h5»Ó÷«ÿôVõèïîUÇ¿WÑ쇎h&TÀ¨1™„ðåªc~u²ø“lˆ²‚àñ j=!…{€“*ÁfÒ(*j—Ëõ ³eM ßu€Ée@ ÊeÌM`[Ÿ\öNõ]cÖÓU%"\ž·¼ç: c[ÖTß3^‡uuðxÓ9å÷}ïÿËçÜrµZ½ð,â;ÙÉN¾ºò?þÓ¿w£û¤y@§j¾*Õ †pü:37Õ(± ÔÆsÊê(ÃsÄÓ®}/:îÔ!Rö «7“ÛÅgòýö[p¢#9© ;V0ªFŒ£ ú #Ç@ÈkLWo~"Ÿm»ù_6`¤o²ñÃ×Ñ>Bê½ðl¹¤ûÇGôãûŸÓÏ>¤g''MbI"êùìi÷®@ cßåɉc]N +lœ®Š¼ %ÈÑa·@‡j‹;¹—…1ò˜-Ãü¥°“¯¬%·yw,¸‚¡Ð¹ùs(é×Öp›Œ2”åQÝäÉ¿Ì›\Ólðï I?çççÍ¿ 8Yç‰îóFo2ŽËï_Tßü<ÄÑò|Ïüðpðøº'ÿûØýs—ȹ IÀ¼n°Aóæ'c#N”ñÀÈOEÅ0E€ªIåFËy^)jêzÁúÂlìMÇo ŸÚn Ûè ·é»_9ç®fr]avÕ{7ÜÁßüÞÁwÿæ÷¦ßýÆkƒ*€ƒuﶽ1R¾ƒå{ŠÏ³Ž)RöS_ßQ±Nm À”mÚtì ’ùr¹¼xYßÉNvòç¨rtÍ´HS `3Š„¨ù”–ÕáÀ(Ëç&–l«Œg%ž/Ìø ŒCC‰& {cáòlCH¢A +ÁvÀ~ò9œ Ä 4M£w¬ °6Ê QSNÂZ½!šë¯B Ç§'ôWÒ>ý„>{üˆ.æsªë` dGöšl”ðB’Û+§Â‰.µ?yࢂ(Œí¢Ç¡ÏœùØ™ã 7wYñ¦ðƋဂ3ú:ÁÈ!Œ¤ï%¤„ó68¸(;ÁÌ0ؤ›ðpÚ\Næ—8ìøŠrÏŽ o‰Ì¼7áö@âòHÄŠ¯˜€&â˜D½W j`89†ë”9j pãìXñùåëŸH„F¹´œ—ùú $LŽéÙb®ÎÍ|- ËQ$Ãvº™Þí CT0ÎbWQro3 *>‰Ý•Ÿ¹€õªxb¯GçGËÿš(¸¨€0A…§÷‘Ôˆ ˜ðÜÎ /k6ÁüàR-X’?•2ëN’¼FW1žÒjõRØÉשV“à8uÎmÔª¯«t¿ê|ixmcøoÚrÝv¬]úþ¾Ì8ë3`ûÂúÂnR¸ &p-Ï¡5ý°×ÿU ­¹L|¨—ÞݯâéƒI5þ“¥£¿]ø•ô^Ho‚V»ššÍ¯ ± •”XØüÊ8ÅrÍÒÄmIÑëVª)飸q¬c“ðç}†»Üw‹°’uï׺yP2¶O¯VJb‹ ]¨¾#×a IDAT>zôú·ÞýúáÁ𛩊ڛwǧÿÆ"Þøtþ“~²øÙÏž,ÏÖµe›—ò=Øõ½w›î½©=W ³)å%›'Ë-ŽÛÉNv²“FÒNù_‡ð9y·räêµ…°’²¬(‡¢"jÃNŒá ŒÜËßÅDvf{MexD²×/iä iþõj(»jçzðþbèDßJx“6@H.‚;À!|.è{Ž¥Ax6X$þ?ë!ÐÙbÑü<8>¢›{Sº}pÐä&‘®Âšupi%Jï;|Ï}‡M‚|qhÛkÂÉ<-8gBÁlhžˆÇ×kbVí68CL‘²Et0LB½‡áFD;+-À¢f0C Þ‰þyL¨*Á‚ñœÃÜ/𦕉j ð ìŸ$,¥'4ÇvŸ64ô ÆžÉÕã0q®¾/†q$¦ºý _×=Zi¨Ü®¤C%çób¹T•ég›ÀÏåP<]lšQœ•<íïAr¢‚ubî|"²šœs9Gìí œ¶|iÉ EqC'àIs•¡ü¹'Í£Ä\MZA,V”RvFçË—ÂN¾8Bx’þéG^„¼­¡}€pÕö\f„!бm×Ôc(õý^ÞY׎>OIúY,”øž\ ëÂ6õÇ&èU–† E‹³Ê‡£T vͦ•—Z@û…F0ØÉš ,^paÃŒÌ,Á…ź\ë ½è9zÐzªÕ”ô?ǾßËùÔ7®Û„Xõ[žwwt[IÀÈiØû•ö¾óökãï GþFÑyŸV0c:t7ÞŸVß|çõ½£³‹Õ'?{0ÿñ?[=<[ÖËPû°¬C¼ º*Óì*Œ»¾ë\Hù¢Ä9wvzzúå¡”íd';y%$8ÿOMIþ+Ô„{$Pô1±ŸÉ!À @öزáQ#,±ÈÅAz ±™Î°ï]á­e/3„Ð(sÄA2ÄÀ_:±S*•ÐDe°À) …Äç{(HÃF­éC"*-i ©ˆÑð)ænå'õÅÓó3zvqNã1ÝÚߧƒ½6k*‘ŒŽ&Ña¤ìª3@)&¡¿T{\®Ìâ"Hèñ'QÀ: „ÍD0ê54ÅVÑnRÔF ¤H”Ð…`¦±¸ÞOá¾°Ì içÔÀ:Ô ƒªƒŽg± fb“ÜêÌ"“,TMf×ØØ—Ï™er±Ø~vÐeB‚±ë²ø_.ÏÝ ½ÑûqûUO'{­h(Ü®ºqF«cçüË@ƒm¼Ê´ÁxØd„õÉ: a“\VÐrl2Ž.û®4·;6]·46ù÷+çýÀ…°ŒtrI`NäšþÝàèc\ÅÈ}%6ù¥¢Ã ¨Ý½YœJ‡It$nD°@b±À`úœÆ1“"Äi½ð>ßΞWnÜåï(ë>/¥œ—D]PoÝûº)\¦Ò.^®óþ²Ä°r£Šª_÷÷¾ûþáG7ßôÞ"ìªì¹j0¤;7GÃÛ‡Ã_øλáñéùò³O-?ýÉãÕѪŽóÙ",.Ù-p øqÀ±è\wÝË@r¬^©ëúøââ¢~e´“ìäK!1ÄO£srn‚N1>˜ýa3@0ï F˜5_“­'GÇÖƒ³Â‰Q•¥È‹ÈDD05gšû«ZÇêØX×&b%FBöæËãåª;í±©ji3šŽ ä0à•½ßÙrz“ûØûF`à÷1ÒélF'³Gt8Ùk˜$“шFƒaÃrÐÜiê!ï8’¤K|ܨg?‚QI6h)L*†iì»nþ×ÛÚ{@? ë&{óÁˆ.†ÖÌJ¾>²>Ððæh Ï!7` *¦½à‡8‚ð®Èmlfm1(‡HZß<7·3!äp àše9HÂô%fÕõY/)`õëhƒyÚTtZÕ´XµùëoÓ±îÚ bH ˆ‡øV®deyÜ‘Ü "w·&š4¬µÏ|¶|uÀ‘T­&U" ¦y]ÙlÐg]8Ȧ0”ë°T¶T®Ê¹¬­å1H‹¿*èR~ßGÏÇó¢«\ôÓ oÞƒ½©[<=õìĹ‹Ùjµlæ^IÒ9\é†ïƒo6 _fP„…Cá@ŒV´<~g)©½{ÍÆS8Ÿ D­.røÄ°Ò¶þöwÙ"}²Îß47…p¬c5]vìe ç:Êe’Ž¸Ú¿u»ºýÁëƒ÷ß~mòÝjàoµL=ãÊ0: ë‰=ëi<žø·Ç“ñ[·o~ù?ŒÏNÏW>]>üôYýt6§§³pq:«WeSÖH}Ì“M¿÷½Ó}×_7_´äÜTO~ÿ÷¿ÓG;ÙÉNv²IFT=XÆåœ)ÔÄë5n¬b!{„Õ~Ý[1Œm.“‰rZ YÄ•Rä~hŒ€.àŨG¾ß B{Wú»nõÀÖàïìS¦-šÿ"pN®0ãDÊç…Bw)íONЙ͒ÛnöIë!î½P]d¾ZÑüä„žŸ5 ’ƒÉM'c†Íßz£½ÄE²÷8—re€<¿XÝÄô¤ªƒà؆'°È¡á™û@ õb#CgLè‘?zÐ?8ÀczôB%×FØ rm–ÌÑ …e$²+c>Af!X!3ò½t;ö•Sfì;Žv[>ÃMú˜ÕÐ9Š6%®kZÕ+Z­ê$é¨Ýr0„~•ê~¾p Aðšå€µÑÑêMiiìôÖüÉ7ð‡çBÈï¸CƒÂúíx:b#¬1]0H×ha÷s"æö{Ÿ˜2rM.¡ ·#Å›± ©iàï~úƒü΢ìž!×GÎŽS9ßëœw•”ËŽ¿ ´xÞðžë€.—ÝsÆJy­ŒXÚÓg„¥(Dªö†aüú›q|ï½8¹ñ¹á~ ‹37?ùÜ/}Bó£‡ny|Rð@X5lDt˜%ëÚüUF¨ÅBbŽ¨×Á®ëà.[ÒÕ: Š|(›­ümxeÔ½‘\T£"ÇÛµe¨†Òg|ÓÅËœ Û¼?ý½Cš¾~{øö7Þœ|gºW½Uy?éÑ&@좎_xï÷&ãT±Ð¿yûæpþÍe<½¸¨Ÿ>8^=||\?::]>yxRŸ/VQ:¦¨¼ ð E>“«°R.;þ ’è½?zÕµ“ìäÕ—Y=x4,EŠ¯[Åœ¬#jáÖ# ù8X\gC—oÌNnŒ«‰¤´o1wÚ_D‘t/Æ# -äkàûË®sòEŒ!IÐò*Õ × ôî;dH§zÐÜ ã¤·[#%p]Íçt6ŸÓð´jÀ‘½ñ¨a•LÆã()=þØ>g¼úÖ¸U[N9› ÕÎ!Fb1bÐê°©¡h™Çœ[«ë¸rœ‰ðä2\»ÛnliTú ê¤ëÄæ0¡Î¬ëªD8ÿáÝáží ñÀP3¸i_þ‹æóXŒ×%ªZ9îå÷† Tv@ñ†Ù”ï¼½x±e‹Ô+Z®j ‰D `Z ¦ð/—õâZù{yß ËÅYŒ-cG¯X"m6›nÂfóø$ÌñÒ~¯8•J\ÏŠ†Iˆ¡ƒUò8ÛdX!t²ŠD7bŒ¸i è3;)Ä:8ú´ÛÐ#×GªªJ ¾Þ*Wa\¬‹ãß–­Ñwòûmî·é:›Î¿¬M—=O_HMùÝ&Ïûºë6 ˆ?ÕÓ7ÞŽ£[Òøö›ä«ÃHnØx~2¥jr'Æ;R}ñ,ÌŽîûåég~öð~\=%€„…%é_»¹Nß|$ºE†[ZC Þ×*\¸!™Å’áwâ˜EPDzG$ YÍ‚‹1Øk]xÍ:ÙfîÐ%†öUC}ïØ‹ž' ¥1Fº·öÞzmôÖ{w‡ß9<¾é+·¯lÒ±adœNƒ¤;‚/{ÞÚ„/Þ»Éhœô¸êîþèâì¢~úädõàèdùÙgO–î‡Îº¸‰5‚² û£ E¢W5Bm_'päé+Дìd'_2úñ2¸ùSW”ÿdcM~Ç0~DH"Žÿgvµ ›€#åžÖCZbBòñQÎQ0 ô®Á † 5 æ€LÍçö7/‰4ÙÞÔ  HÔ>v¤•K¼1æ•AÀìžPÂF„1·u™Êé×5ÎgttvN£á€&º±7¥édB•¯Ì¨ Ó ‹N> µŽ5‡>+td b`3«0bQ!7 §Á|(d TÌ€Ø1¦û~×éIx„a¨€NÝù É`Á£ÆL1p»ÍkqÙ8ܯݓaÆFÍ úP´pTdÁÜ2Ø; ÌÀ‡d1 DôÐSÂU½^û³Ô­“~•ÂgV Šèˆ¶õÞQ‹É¼(˜yTðsC‡ës Ý!f6ÔJûE–‘pv®èKù(EÞn^Ÿ¨å;q(ä‘冢&Š&¼]þe)ÞJ¡4‘hd³XGjG¯8rvv¶L^Á>t©*8ÐV²MŽ>ða²N¶ñÄ^fL–ÀƦç链ö—Kß5ñœ>ã7ÆaúÖ[qüÚ·ãèàmçÓ@Ôì4fQvTE78$x@ƒƒ7êzù¸ÿæ·<ý”f?öó‡Oc}Þ $\8$)™+ç'¹ ̹,§Ä+'Árȧ()$Åz$wÃŒõyCŒèÙ1%ç`…ì„ËX‘(<&Éj£¬„Ú…h&å¶ùCp<®ò7õ0F6…ºô½Û´çyæW¡ùæ›Ã;¼¹÷K·ßTÕsÑk¢2eõ‰Æ(SEª+£póxÐ$%#çâèÖauãð`ðÎêÞð—>|+<}vVß?9Y~òù“å“OŽÂŦ\ %cä*aC›þ~Ä{ŸíÊøîd';¹²üýg?šÿw_ÿ$ƒU’²òV*û%É:†:eCÆ—Æ‚ùVïá8y*ì˜tÁW„Säu’¥›S;À‰5ÈÙc:6†Èob£©È;P§~0"'àQ4톲ÜpcŒqu­^£ýç:F¹>ÙQM É¢^Ñù|NONNi:7l’ûSåD®Æ8\È2É©IŽ½E<à‹9HëL?F8¾Èç†nÄÐìxÐ)¬à*0Ê*Ñü%† ‚M`»]æOWÇÄÉ‹Ñ ïO¶‚!³\Ôi¤óÈööqt)æKjävY'­qßMðJÀøˆæùP1Î@=ζ&\6â [Ù5Š¸âIåN€H²{Òï\=R—o@/ÏÍ@‚s¼Øv3 ™ÑTµÂvŠÊš/fâÂ<…¼!îP&,DÅó^*Q;rË5€i&'þÕwÒ®ª/2Rá¼ærÁbZ¥J4û‘èµ@nš#ñóÁþÜWñ ½$¹8Òöo<Å¿¯ËXj¬Ëo±<¸ìúåï´ÈÙtìe^àu@M_XÂ&@¤¯6}F4òqüækaÿ½_¦Éá·¢LÚy¦ >/ʸÇÖê¹ÁhDÃáaßx‡&¯ÿZ¬¿qßÍŸþ8^|ú)Íž¢— RjDMµf“lb\°ú"¤I\í+ï…E„#,ЈÌjr1ÈP-—ÈYÃõ=€ˆz$T!‘ÍÀÅëر‚ûòŒ”ŸõÍ)—*Çg[Ã{›Üë@þk]g>¤g¸;¥½ï¾?ùö›w‡Gƒ[Dq Ú d·Ö “™!xYÈÕë Ê8Ü?Ÿï*GC?¬†£áààæøöêµáGßz—ŽNfËûñW³?þËû‹Þð’>†Ù&y"äÜ{þʶn';ÙÉ++÷îÝKvȃ¦}èí…ÀzF:ôNŽag … ˆ… ¥_ Ó‰dKíªœ ûÄßiæø#ÙÃd²ÐÙ[‚-m¢SÂö€§8–ðŒ¹ f@¦pÀ稢¶l&±7·¨tÇñÿ¦zNþMúúÞ&AÎS¥b± C–Økû<Ýc‘Ê¡Ö5_\Ðý£§ “äÖÁí'6IUÙ…Vø:Ì„¢C8?§S‡IÄ ™j5T€kÌl*ZdrÔ©Mª•q àdˆp-W4'MŠp$,qKÁ $xë§ÂŒÉUWY#å5€ÝQêL@‚¦³íc5L¤+àgszÄWt Þ„û"^fÒd® Yäêž ŠÈ¨”¾Ëj+p8Œ–¦!hRæ–‘žÒå._”÷ÒKÂÝöÝÇw˜KàRY-³w\7=ÐQz£jŠ÷.È`ãñv^J5-i+"ɸ’›´!4ñVh 6 £(ì†gU¨^Yp¤)ç{Ý\ëÀƒuÃ:æHy¯MÚº6\Fùï»N»?_w­>6Å&/ú¦ÏbJ{8~çÆòàÝoÑÞí_&?¼e5ìÅ/àIE¢]r(Œh0Ñ`|Ɔé[ÏhyúöɫŧOÂj^£§Ãm¶ »yU$)pŹ†C+›Áæ¤{b7‘nøÐeÅAa±%éìñ°èfH³&†\Ì|›ç(Vü>Àƒ)Ù (}ïä¦yÚ÷”ß¿ˆ±oÚO+÷»ƒ;}¸÷½‡ƒo;çFê…i³tCÊ­^§-µF²\‹’É]äw«j•`Iž IôãÑ`8ÖÓñÈ¿~zÇ=þËût´ô¼ käKŒ$™ƒù+ÐŽìd'_2999qáàÆ#O1Q§sŒ…õë«#éû½ÚoÌܧѤgÂAè ÆöçóK¨Ç’—DzmÇe¡CTÃC >>t‰ÙX ¤Õ52œKÅ;Ôã@Ïê!ΖÎn¨¶ØL!÷vYe7Æ(ìM2ËÏ€,eëÈWƒ¸1Åa @ T§6ÔDOOèéÙ)íÆts:m*ޤߛ¢¨Ç‚Q‰Lt*Ãf“+F7”ñŠ› ƒèÃÖ Y·Bc¼â0qN(•M„ž}§:(Lbíçw€ÍTHqgÚ Oa>»FÚÑAòwQ qI@Š`°|ÄÞ ž{ò=D÷ö €  @FÙ$|4i§u^*©íhÙòMMr¶ˆe¨ ‹Š‚ó#tõĨ8aÈ〩ç5É¢™GòΉ:›oÊe£ÃêJÎÜŠûÈc¿8,Œc™Õ2É<žô½‡2×æq~¿¼€…¹¡é•Þ[dß®{¥ƒ2ÿz:ËzIr-pä£>Šø‡˜þxƒg'y]hͦãÊch (±îÜu×ØÔÞuíÚt½uL‘M÷ÄãËcÍ5†7ÇqúíÃþk¿«á[¸*àKÔ Íh¸—Ô4¤mÉ Ñl½Œ^£ÁÝ»a|ãÃâßUóG?¢ó¿úŒ–'snË2£«>W»IIb”¼ê`H).ºPîá(bH¯H@ÐM7êz¢ ¹3Yð•Bf.$Ñ™8}¡56æu½lšÛt…ÜÛ°61®¶1ùœôlûÃ0úèýɇï¿1þõñÄßMñd¹7 !šQUs¿P³`ÇL6@ {Š„ºú·U pìË~N »êzéBⲎ«¾¾ü’W–ãÅþþþ®RÍNv²“+Ëááa¬èãè]²N*D‡šèÏ#±ðr#Eª@°'˜i¼²EÎ Ï•_ ôÄ‘2¢xp©“„R€²@IÿCZü×EM$k¶±ö“@ʳ±ÁÍ+¸oPÿ)€&>¨e•81⌟ +Êc•ÊŸÎ.hpôŒnìíуCÚŸNiØ$EòUÜ“éë¹/d‹ÎæJã™HŠè}Ëœ#hHË}¼Uò¬©^$¶ŠM¯°nƒÀ!öCFÝôþ˜#¤ù;”€´‘ádžG›ÓC³ë(P‚yE"uÛ ¬ˆ\¡È0Â`¾0T>'˜2ë%Oà(O }°X.„-¢÷Чµ!òÐn&òûƒö:XO´Ê.GµäÏaC°®¬#IßØz 8AXŠÃ{¬‰Ì<¡(ó…ía»óÈ –X&gÔIJ¼ðØ:—¡}îDrw r Ü”Až<'¸ Á=‹Õáì’‘¾¶\›9â½?Ké(¼÷ÕUs~ôåÏhû2ºêöhëê£åÜCE>&ƺël{ßây¶ú¬”Ma>›À•mò;¬ý¾:ÇÃß©÷^ûeLÞqÞ©³Ð·çûO { µà¹à$:W âäð»õøà[qÿ­ûîâÙÝÑÿûgŽVŸ=ý0X’I^u‰²cèæ%ÿ‡Ï‹ Øc×ü‹.Ô%…R6<Ð!˨Ü~Ù‚¤ÏÖ…rmd\öÙ&V¿.4®lÏeÂ,¥‚sÃxsäW¿Âðõ<µXƒ|Åä…^HQ™!@uµïŠe„èwª\ÙÞì?)×õRv¥¤¤‹:¾”2c¯ºÄþèþ¨þr?ÅNv²“/BîÝ»ãçGžÜ¨ßÑZ%s#Bتñ2DóJêÁ^`Ÿ =0¡¾k1èáRt‚(·¢¨ñ«ÏbX§Î¶“÷'=Ÿ˜v?«søŠ1ÔÀÜT©È,Êw ðá0Сš–)V[Ù4ª?²;ÂeæMãÉÎGèWfîë1jÌFx3Þp픟äñé)ŸÓþxLwoÐÍý}6Œe0Ä‹Ð  Õaݸ¨@NÖÇx´]ÎÒHX§ó 7*H‚Ž•˜-YÑ9 ëU­iö¶—:&2ŸŠyCdûA_"¥¶N(G&9­3÷Ã9ÜÌ1vöAY ­³ÜðDŸ ÞñC!fe‘è»Âß1hÖq6*…rMúå|¹ Õr Æ;š¿ûÂDPÐ@*ƒT:5ÉàÞè`ðîôýÑÍñ·cŒËÕƒ‹?ŸÿäâãÕéb¶.äf SÞSÛh 0±M¾ê1×ý¾)DˆÖ’æ¹âh÷?|3¾ýkq°÷Nbð+2­Æ6þ‹9XEä ã.ñ8µ“Úç£XÞ«Ç7ïøὫågGØV~ÆDKK?óT¶-ײOŒ’W(iß:Ù5Œ^e2ë%pà2qÄÝ}Abš 6I\ÔÍ&ÒTÅÄîc/˜¸Ã"¼,}·)Œc]–¾Ïûæ|ùoyý¾{l gkÐüÅBŽ‹+9>™ÑƒÝ8ÑÁÁ˜•×½s9n–²#^˜÷f¨"fõ.3~[ωx‡0X/B²ƒË×u\nêÓ¯ªTUuþð;pd';ÙÉ•%±“Ï?þ'ÏÜ+?Ú’³è-eeY)×B!&ì9U6!8Ѹ6C“—Ãq. ¦v‹ÿ3ßuBp ÔÄ=¢FNdÖI‘ïC˜0ìªq¤í'ÍgP#úÌAº ‡¾}c˜7h¸wðÛ£„÷Ø#åeäGCP&ÍïäÌ],æB†Ž3§ï%E«û¡]`Ø4V,1feÈz… ÌôÚEH€+,t/Ú¸M( v8f1À[²¾rñÁ!sò à Ö7„tJºü|ŸÇ§yåÓ‚¸)¦0šÃØ€¸Ü¶ ˜•eñ¼H~l7 'züpÙ\/F®Ž8çâoýÖo„RÞ‘1]bì£KÄEç+_ ßß¼9}otkô-?¾Á½:|üÞè­ýGËG1ÿøâ§áxuæE½nÇá2@d[`cc¤4"×åñëdݱ۰b’a†¯ß©o|ë{n|óT ö=Ìô.ûbò8`û¸VëÔ¥ÎÂGSN‰b\·ms>:ï7õÿ›’$œ—$%üwßX~aR·eHtsæ~„ÒhD²@Éê ¨ xê´‡I0Hà¥6Ùñ´à«ãQíÙeb©¸òùŸ“°®“¾9².ô¥-UÎãMáU›Þ›Ô^®”dÄ…Dˆô‹EMŸÎèѳ9íÎè %#šL†4¤w (¡Øs ×ò¸:ûw¨írCßÔ3bXQ¨WM|õ5 y=ŸÓâ²çüª‰sMø×c²IÚw²“ìd+I¸¿ó÷iÇ àÆàf½ÔøˆÂî`à ~Sò }Þðwf¸R‰õæáñÒVÑ|ĈÚÖN Ôõ4¢âÙ¬qQàJ ÈØ7 6-WuFÓêÆüî@˜˜aûðzbÇç"¾°jWA%¢²Y‘.ö}4(¿Bi ÀŒ‡œ&àXy Kµ¥tðÄ1(¢¥˜½\O>>ßßߟõy©QŒÁœ«rÎühpg|küÎþûÃۣ︪ºÓ$Èi*K~`£=Ž'Î|Zã}Ïk8†]aKÆHŽãHî&Q¼R…¨ÏÅt¹4ç<3¼=ÚCí±‰¬V}J/Q® Ž¬V«yªX³Éà2†¿'?˜ FþÎèÎè½÷wFߨFƒ»9³CðòäYêÕáéíñ­é/¬Þ_=ZÏ?]ÞŸ=¨ÏÂ3:]Ñ2Ô›ØÛ„Äl“‹aQ·Î›Þþ³IÖ IQˆa–çG´¸8¥j8¦j4¥ÁdB~0¤j0²õÊE¢n*Þ.*åQ^=ô4h¥mKþ%ݪªJp§ì—M¡\éÆçD®ÜW_XâUÂ[‡TAë3ßÁ¦JZOþÎç;@÷,‹“Q ŒCŽ‚7²é½ë“KóØ•h†J*뀢.ek¤O2KªŸ-âBp-ã¶P^³²y1¯›Ÿ'GsšN´¿?¤ý鈦{”°¨WÉ‚%2v°É‰—-±E ´Ñ:ˆ¼Žqµ ôµ -q.%Q¤—–E|';ÙÉW_~tr|ú+7&÷#Ù½´T9^ e"h¹ÛX-– …9†oíÀÐ_îÇ /¥Ü„gZÀYªDHï*mJ­7fnèó©·°ã}u9oFv„ÕÚ\îùL‡ê4•’Ï€’'ëœ+5±íe#ýS°b6ÔŒ,ÐHšk…C•|7_­èáÉ =9=£éxD“=:œîÑd4n’¸VÞ˾¯×†’=ì[nùÌÎ:ÎÄZÔ‘–»Ä(@†Wƒ9nÃkð«2äÃ3ø‰°„³såÄ5áôÈÈ`¦ƒØÛ:Ì|£\_5 óZ ÖçÃôŽ&ÎWý=ó–`8SÄv=Hh’ƒve˜ÔFÊ!ö«T /î©ýÝÉÝÈ#ƒ`”4žÇXÿÆï  ‹† *ò¤$RÅ€‰?1÷›Â«ün:­5u®zx’w æcDÆœ Ù|6tD“Ð$^¥ƒ˜Ê}ÀÓ*0ÔÑx}üCj<Å¥t¿ _Œ\¹wïÞr¹\]fðû‘÷~âGսɽÁÉû£[ãwýØß!r•}±º/³µ$Á»é`:z°7|{twzZ_,/Ï?®Ï?_/Ÿù Út™w»YÇ)Ïíû~ÝgÛä()ÏI¹¨øE­—3ª—sZ^T-82šÐ`´Gƒñ¹j`cY0Ì¢]ùˆ4_%X.h ÆxKÁŒ.¸ªb’EÙ·%›Á¢²ÒB•}—“ˆ~aùI¢-Æ…]&*ü®\Äñ@ü¢³èâ&Ô“¸ÉÜ—ÂØü‡ŸoH® ž” IË©oÞ¯Ûò¸MR²E.]SU9á½&Ÿã/AùL½|v±¢Ó‹ ªíM*Ú› é`Øü›€’ªjÃo˜Eåû˜ãZ ’€‘( a9nˆàÇÅl¾vàHŒ1%Ó>~š²“ìäK*?ŸÎ~•¦Ÿ÷±ú÷b,qYË{²Nbi¸«wn­ç¾÷þ˜çFEëÌ‹!B{úŸÆ~ņ¥U6ÀÀFg‰Ëí^æÍEX¯6©M-aó©”ý‘Á~Fa@Ð%ß"H‚U›ð_ºA 1€<°/ô«#¨†Ñã×3¶~¡›¢ç#% M•nNf3zrzBÓñ˜öÇ“†M2Îœ3<àšÑèÊD°Æ3À`móHÚ_Fq:OC@Øή8–Ó05yyÜz‘=œgkRôB×ØÇ¿-k¯©o„À…ѼQÖ¼€¶?‚™ãå½qMÈsJ”ŸÂhV5è˜}ñ.ÈîpJ¯GEÿëýqd"°Ž4LÞ­|¥.‡Š×!§EloLçbÌò£ó®3jü–ðˆkŠ‚ŽÕ&D}7±ý}ڹġ[‘â@n¤j²+rõpG;èœ2B ˆtVú¼çŽ/L® Žd9iX 1eŸ(ŒüdwV£Á{onŽßܽå‡ÕMr鞆Ŗ2‰eMrnà‡Õ-7ô7‡£wÂ[Óg«ãåçË'?[=š?'õE‚µûBXJ)™ Ûä Ùf°î¼u÷îË?RžÛ>|½ŒÔð÷õ% +ZÍWT/.hY%FɈª$™Ò`4n’RÙxÃZßÇáЙN&iÙ€Û±ðÑ7œžmú¡ì³2¤ˆrÈ KF°4pú—ó“¼T >DWG©L#O^`ê¸Â¬ØRPÁé‹Y,<¸7@¬mõõWlbðlâwt vUáùÆã¿)”MîçhÀ9ì÷+­Áªò¿«:ÒÉÙŠNÏ–ôìÈÓx4 Ñ¨¢ƒé€¦Ó!MÆÃ(JÓ5ç„qĺG:S©!´aÂj±úú1G¼÷óº®/^¦ìd';ùKŒñ¾#"e&‚ÖÌf(öRó· £!aX‰ók.NTÖ û¬s}©î@¥5`0M Ida¤††{H6ΓfEmd$-,!5Ó„“¿JÇ$qÔæsÛ´Næ»0¦USíÇ°bÄPtRHc×ßç½}‹ìŽrŒò|˜/—ÍÏñù ÚŸìÑÁdB{ãI®xƒ÷Qû„¯¤`‰†Î怀o8&0=°ì4z¡õ:Œ¶¢!6¦áºò°^Ã_0ˆéCgú‰Ù:UàùDÒZ…Ä&±[¢®bi CNhö[´ˆž2º»C¯ã€l¹\ (ÒçÈΑm@„AŒ¶ï„M¡p?zxg0ÜÇŒ °JšÏ½ë¬EÀU˜þæ’]0´ /W0&¸ ãGÅ OxeÆx¸ßäéc4ï~ó¿‘; rwB“|Õy¼ŠºDg¯ Ó»*_>D:NÜK ݾ¶ÅùðáCwxxxì}“–I iø±¯üÍÑdüæøƒÁÍñÕÁèuçý^ Š°°BPþÊÖw0È´þEH…n&~2xs<¼6º3ùöêíåÃå“Ù_Î?=ÿØ?]÷y´×Ñýû˜ }žóu (×1?Ö%³¤ J؉´täZC Hÿ[/iY/i5?§ÅÙQ §‡4MÈWƒìó1oÊŒä¡Ðîdœ6´•¢_×=צðš¾\%üÜœD”¹¾L$õ‚#¨Pª¿¨Øïv]„…úÖróêg {¸ õS::PÖ1Cú%ëæqŸ¬MhÍ|^î•÷¥žùMr‹¬kc¢ÏT>½€ê(Êz‘ä;çh±jkÙ» GǧMpMÆ (ÒÃqÃ*ñ•övªBS×+ÝÈ`'aJo»é±2ó3¸Ù|ùµLJz~vvöÒ²ˆïd';ùzH úÜQ¬›™!ÔrÞ3¹ÂLÙ+n#TtuF eÐîÓ¸w"r(¨¸÷Ùï|<ï|!Ç蓃#:M>È`ï)ŽÉ2|ª”õÍ¡(Æ`"8ÃV¯ ùy42F;Tõ`£Î&glïe%ïIáËð'ùVÐÔ"1TYß2÷gV:ñ¨{±ÀuV—B«/‘9ëe ‹Å‚Nfó¦ÊÝd0¢[ûSº¹@ãÑ° SÇ-€ Öî×Á\ÙÒ$›xóuH€±(ÖÑ1Á9j‘…öÂe¢3:¦jBz^'HNFlÕ²¸ü7ië:vBÈ€ÊOøfñùÎa{ kÁ.Ä°«+aØ3±E’~¹\­¨^ÕÀ°!nµ!\|œ•$!Pò˜¢ó)C£@5¥•>õ/´ ™ùip®”ãÁ(‡‹mE{h#;°!ÇPS,€ò8ð{"æ`n›‹³É&æþÁrßò¤\fœ×™T,jBäîŠ-HÂmç×ѹhÇ»°üÙ.jMæÚ¯sO«y˜ÑK”ç²4cŒι•snDòÕÑþðÝé7‡7Çߌ‡·£alÖÏ¢<¾Ÿ&fÏÖ@&§è¢±ëåW§‹ÅúƒÑÉtx8~{”’¸ž,~zñãã?ŽWç´†ÑÀRæ)Y$› Ê>cÏë3×]o±é¨^¸X¯bSÚL'‰Ä£ P’&Ü’çÇ´¼8!?Ópr@ýý†YâZXY®Áí+C"PQ;q²¶P­›Øæù.ËÅRæ½àD® $á2Ò/RšœP9¶¦Kåå @þ¦ÈK‚sšÕ5Ô-(Õ¡„öüž¯Bжõçœ+6Žn¿_—áA=¹HÊÏ×½}ç¥M«ïZ—I³ß{§ÊÆ‚YƒrJ¤Š2x"~.c–Bº­V‘æ‹šŽO–tÿÑ9íï éî­ Ý¸‘Þ—&볌æ! ÅÁ4Vå2̾FEjDêº>™Ïç»2¾;ÙÉNžKÑ3O.ÕHªÂ@€ &ùTƒOÙi­®²A¯{s4Æ„Ú=Uq©ÂJ{QC€‚'0R¤<&$Tg½XXdí¾~÷Kk:ÉÚj49\FÈÀ À²©Ü&Oôl°½"!œ"÷³\š¶,§ôSº?³²aHé>±©|“ v¸¦±ÀÖö¹[¦ ¸žØh#>"<4o=c)a ðI¥Z,SÅ’Ì/èÁQ›ÈõöÁíïM!?™†Z(HR$á‡ùØãcŸ²ž@Ð0Ê[+òBh cÖOqq@lhñèd6gÈOcÚØ“`5W‚ t}‚¼ø©jL¶= Åh†(÷ 8Ïͽ“Ç$ë—óÅ‚êP·) ¤5·Fô6 ~g£^PÅî)ªXåñ2 -@9χ‡Ë8Þ.J©\‚qPÌ£o¡8ôt–ûª³Þ9òæQrXŒÃfäv¸–ïȾSæ½Ó»äò)šÈ—¯ºÃ@ñn¤8mcüô©ó\ ’DL ÐM˜äÏ‚só“9½Dyžœ#q>ŸÏšþ»;:¿;ý`x{ôÝjoðšo2+"èTdŽÝ… <¥™×,ov·È¬~±©0ÇÕtð†U¯¯–ËÓÅã“?íc*ôy¾û¾_N€rY8¦ðšm˜äÂÊ¥òîe˜Ïá-ª]ZëùÕ‹ÍOžÐ`¼O£ýCL¦­Â¢¹.>qFeAçƒóÃêã2 hSΕòXd±XPò¢’¸ÆØ®nP&䢼 éÃ,q-oÄ™S6€ž³„@Œà5Ÿ%ämµ`œ”@H«7”¾ï×w}ã²i¯cbñ÷©iì®+ik¨|Jí™l,-ÜÈËþp˜ ?+pËHG«œ.éÝ·&tûæH]é´ré^‡•§ÕÊÍ¡öýêKUUç_«ÞÉNžOþ3"úO‰èÑ? ¢ÿ;E~Ýût2™Dâ1ù¦È¾î‡)¼(ä Œ F»8‚À° ±X•M…9KˆœÝÛóM’Q-†à8b@†‰\ª2ª€}¨ ý Ð‰ÎÛ”¶†(38PûŠ1˜6ȵ%1‰+ÚP„û´ýÁ` ³VëṌo@ÍúŒ=ÖRê·ôj‹—˜´Ò…W]ËCèŽOœ Gà ý´4yˆß kÊ}Íâ’fÇ ztrL‡ $Ù? ûû4Êz¦Î[’W-â%êC3ö êë¦ë±´º, LPÏÇð3–Æ·ý¾š„"ö(½Ü^l·t)äY1Æ/íÆn£Xè´Ex ƒf=Ž?ý»íçdÓ$VðjUSà‚ŽÁ `t4oi ü-tÌ]vŒå¯0I*@Š ”¦I>dã¸âûqÁúD('óûÊæìÀðšÜ‘f­à>¬¢¾ƒç‡5ÍÞÆhûu‰nGŠwùQlJb÷²`½ág¥Hâ¸üΣә+Ù4Ç?¡—,×G>úè£øç÷ÿ¼Ž¯UïŽ^›þj2|‹œkÂ,"ð},v¡ÙsmÕ¤Lã_ÚJë’!ЧôÆ ê¦R½ó“Á¾öo“3aÝg›’´öI_Xκs×61®ÈÇ¥¡ܵH¡GX¾­®±¼8¦å옆{7i|ãnºYªrmX@e?iÌÔ!¶sSOZJm2¸‰ÖR)#ý ›äy“¸úÀÉîYIb68‚p îy§ eý"‡Ká΀sò×°Á§ÏS¦ 1 s|±y”ÇnÃXºêÜî©g<ÓçËón)© £F7–Í7v^gòCw.*¯ Ìëò÷H8.e§ôgQ†[OÙ뵬ãŪv_;îHáx<ïÂjv²“í$Ñ ÿ­üó":%¢’’˜üñe8úWQ¾ùÍoúäñùp#6ëq6ˆX0«Á)Aj@Ë1s^t{B¼]ÂK.àF>ÞÙóTeVoOs”{ǃ³0bYà’­ÛÑ­hômñ+`ŽŠ\6hÐ{Þþås.… ®m¾#'^gìצ„oÖ0$p[¢7 dêB†MAÒ^®üÁ«N¸¿“?w8Þ’ûƒó¦¨GÃÁØ£ÉÁºC#]\ÐÑùž>¡[Óý–M26ÕnZ;£…‘q,L¨%‚9qĨ.UL ŠƒèVŽÙ)ʪ^³êÃÞ£s|0˜ aX&±—0ö‘¸ò$ðÝ®tzN©×Jۜ·:Ó#\…#HŒŠWæ]RˆBæˆÜOÁ?SvÞßP¶›Ûá´ÊSøâ°áw@B`lx?Ÿ…}¬ù~ðžöùœN'=?3ñjì_R ß)„:Õ¯¸F‘ènhÀd›>ì*óö“¬-!BhzÊÙ`sï¹}áþÿð;?}uÃjïÝœ¾ñ½X¹÷ÛOÐЃOëD;y`¡Jõ {jÌÀ@`L6ZšÅ¹ÔŸcŸÜ(ú˜¸a“w™a¿¹ŒõQ^»ï^xuß;³ÚŠHc$†±A¦3z//y²›‰™M Òt2õ€tºPÒµ¢÷ClwŸQ½‰Q°MHEidãø•l’Ž$„«Ý\U¢s‘=B­ ýN=ýÓAµuCQè»Ôbl— ý7/ÚE=ß>6~¿.´¦Dßûú|[ÆÔ:P eÛ*4ÛŠq€Ïï]±ÕˆÆ‰ݺì;JÚ_ê`¥•×!L›m£G=(ÁHe u¸>EæË+¡ªª“Ä,ü>ûNvò"䀈~3ÿ$I% ÿ€%?ù:ôrrÀ=©þîéxvs–òCÅ95©Y¸¨€!¡†Gé‘·Æ4ÁzM’x^se¡hn1b%÷†e4”¡4¯ïœêr}0:JÆ£ëÆùk³|¢ä¬xv‚Ôw9”Ÿ)% Ïà÷—¦4`š¼‚CB¿§˜Ëiª£€Ïubà8³'3cÄ8ÆØeã2ÊóÄJÁ9Å(ÍK¡ “LÒ‰š¤r6Q*÷±+À /6CöüƒžZ’Êß?>jØ$ûãq’Ü:8lØ$PØÿ~íÌ•Û+²ý˜†èdfèŒ=€<àp#éSž`;­  FuÈn8†öد1‚¥“¯tÖ9d<ɳ2È¢ãG¦Ÿìüí\3?Êr¹ ÅrÙ&'î¤ÕÐëy™o–•¡ó "É÷¡ 0J¸ïu™ÈIX£ÞßJ¡Ô$ >Ú}`k´Ó)æ8@®³@ )8ƒþ>‚ĪÞ{„¢¸K&zȪ ^!¬p¹}q?…“Š‡!Ûñ¬“.èÑÒê$/Jå¡Vhè3mI_r¡Žä&f!½D¹¶›ýûßÿ~Œçq~úôtþì³Çtúô„–ó•5¼ qhùâ[TP'Š| ´9‰Çh|MâF&ú}µÖÚeaë ñuÇö…ËàwÛäÔÀãJ6 þî|X¹° ú²c/Á"IŒ&€RCA&ÙÔ1–ñ˜ÂŽ–áÎtP_1ˆQ‚@ëBpˆºaåõ˜}PÞ/}~qqA³ÙŒÎÎήÌRÈ{<Ħš§nÿùD 78y…‰×„ù«·°ýO²PEÊD…P‚Uxìægë)e8R_BáMãÖˆ”×਌ä}bC”ä¢\H_ Å».àu çP6×)ˆåÌqRþ— Y­~¢„À˜IųÛl¼Ì2¼Î è°:îâôÈ¡Ðh,úÌ&I%òèýù'ŸÐgÑÅl–Aœ0õ™¬ S‹ç1è%ªvä?ñaµÍÆè.ì!<ß9ýLƒxFÛg˜ŒUÚªŸxnŸ³Ðèeü$Ee«¡¯J tÕu ÙbN³ù¼ù]ìÖø–ë˜ùM«L^2C" „2ÍìS©Ê¯+†£htLW :Ï5œ#å92IŒ]TÖŽË×tž¥<«#)ÐQ3$ìt<µtl/“S™.ÞŒW š¾nå(ÜŠDL¯"R*›ŸT³%µÀ{×üðs3XÒæj²`ñÈ÷Ì6¡öZ>ýëG‡‡‡¯&8’Jøþ‹ßû£gãÃzQÓÙãJ ÉÉ£cš]P½ M Ÿ]ìålbT4Âd7# Þ[¥Öƒ„†·±‡¢ÒýˆF¾ÒéÒÇbèû½üŽz@‘>„¨Ë¤(Ï)¯ÙjR^/¶3Ã$ e4Á(]ÜI_ù"./6eIa\JcÜ$È\›(újª*S!å32‹£ƒPÊçOç¬c,\:%c|>ŸÓééi’¤¿·JZà2oó‚ÎâânóMðÜìRíu›° Ê­ Ìkú¹ÃÜñ®JYHqí2{Ö1}8¬¦¥¦uÙ7}Àা§Ì¢bÞ§û$`äyÃhºÏ!N*Û“NÀ?-§æs,Öš"'Iz$/Ø_ûþ4ËLf¼…üð ârKGË_j¨WQœsaµZ%Ãæëöì;ÙÉÏI¾MDÿ9ý6= ¢? ¢ÿ‰ˆþ="š~U!阿÷GŸ4:&U‘âA z=’#¶†d˜}X„p ÑÓ^¾cAÖƒaiDËAã.Rw¿ÁE¯5ºáž ÇòñhÀ ¸ï€1ÿöhì:0¥ŒÇ>†Zlš:6ÊÁoƒºŸô‡³Ïdž¢”L#–Y0ìi“ê5¥‘\&Ïu yú„þüÓé¯>ÿŒžÑlž€Z]A‘®>¡]œ°"J&utÌRZSó€h*‚"õ€¶2  Ô"jô\“þ³MÖÿ„sÖ>“†*É\Aæ3¸[¼^5ÀHÒ3õ¶O`!2 gì“ IDATIåºÆÙnŸOØ%fn´ß{â¬ø¢XCä  „El ¼ìb¦š¬úþãû…Iå”$Qˆ4ñ±§g%¬s~ä5#…%Âmâö:h«ã3SÍ›â[‘ÂD߶ë<.ÕËé4©¨ùi@׆Êè ðšˆà·M¢üo¾ŠŸþÆoüÆK Y®,–ózÉ=á>©—+:vJGŸ?¥£ûOèü蜳E’˜— ³àõÐÊä…(¼Äaƒ^dÉâMä~JU} ÄÚNÙ ïEP¶aú®Ók€&D#Ä /, ë âò˜”Ò%”‹þ­H§Ë9f¶aÈl’uy.¶ÍÒÆÄç&äüü¼ùÙ(a°£í3õ`‹Ž W¹oÐcÁæ#"lš%õ6Ü”OKc©Ûþ@d¾¤Ä­ë§ÖÈï"4öy%p·‰Eò¢Ù"æR i«rÒv3¦cÿ*âYñ´n « æ…±íõ&u”…æ˜vƒbO˜ÆÜƸZT_»Š-©‚YªVã\é'ÙÉNvò$­RÿýWDôÑS"úÇDôßÑßxÞÐé/ZÎêጢ“ä{.${)á_¤x7»2·FJ–²6cÕ0.¨°ÁlÖ&p\¹'š´lö*|,|‹c”Á:€ ÷aãËæÁö«¡£†Þ÷9ïXWÚirVDÔaôY}ÿ=ñ¿âËP¡akŠœ¬£À½!‰lÛ<¯F#(¹À¢dC•Y2Ò'€Õè~¬F.÷‰:~9ÆXV£Ÿ©Ô:R™Ø‡''ôã÷釟~L?x@OOFɪ^µ:dƒ“µ`½ÐÇSÀ†‰z½&ü˜ÌЃìÄŽ£$È.Ça¼dÑuæY„Ù,`Qz˜“¬®­j›Õq“þžÀ©yéõŽ,`pŠ{¨‹À᥀¡80`‡°ºÈ0/?}¾.Wa Ò¶  nl‰>‡¹ 8ùJbÔr¾üÚÄ dxªÚ°ôözò Qç¡S 32»K®ŸùÛ‘ÜüÝšªq]™CÊš‘–Ü£9íEv.6ø (m«?Ç4ñÔ„×´À Ïhçzt§U=xø²pϵq.÷—3GñsŠ®NyžHgsZžÏéb8 Ñþ˜Æ{cŽGäÔ¼æÅjÑ„Âæ¼Ñ/©AGA rêiäö#z¶šõ…¯\4Ásúò‚ôåh¸ìº}çö…”$8HƒAmΓ§‚Å”“m’ÛH¿´–_NÙœ.P¤W×{¹ª)©ãúç[Ǫé냫"D]æ¦c¸_97IÊK’~Ò÷˜Ÿ$RðÊ íïcÜ x£·‚6. `hãñŽÈÞÓÆ{æMtcÇ”¹Hú@’¾¹Ú'ëŽYªðßË&æóêåy¯*>ºD#¸!™ùUÉt²Èç¾ÑNR%º|_\¦ÿq¨Z»Ž* _2„س‰¦{ébôÞ›ÃÃ'ËPÏOÂâtW_°d~xxxö ´c';ù:JJîúùç¿#¢c"úýœ«$ýüÉ—©Oö—˹¤cú:©i÷ßdáN¨ùIå"ÝÜ]8r ×Ôùáüâ4@ <òʆP6²¤ƒ'·LJ°g£-´¢†I¼,Ao 0È9MÅhFоë@¥ý%Øp÷&6kÓÜB®:Ó|çщ !J‹Ì—ôa¡Ä8y.­:)†…”!Ðîðü{y.N^š¡ne`˜{†W„•¡ Î…HœCÅ&@E£ù|± ³EªtsBÓш§{t°7¥½Ñˆ†X,€ wç̤ÑðøžHh?vLU§ï€!¨ÌŦœ ±çoèé5@‚ê´Ýœ…jKŽºH9"á 0Q±ÿÙ!kí„4ùëêUÃÐ1ç@î°„¼+`NG¨‚ ¥Ó±¶¾´Ç1Bïñ»©À…‹ÎFGçl¯ÊÞo EkcÛpmÁ¦ÉŠ¹¤¹”4Ý‚Tè2y†´Ï # / ß°±@Rš[âÍ–¹×ô¨ûŽ`ÚáPòn圣òì&ð#ÀcäL=\ÉJxFÕüÉËvÀ=82¾/]¢uºTj­¥t±ÏˆÏj±¤z±¢ùÉŒ†“!¦í¨Q4I„€ eÔO‘u¤$Ú¿sR¡Ä™ÐpHQzÉûò'PQßÇôèKbyÀ…Ö€)íg À‹KžX²¬a/f*ª›|™Ï@ôÝ.ªZiˆß¬&©y°æ¹¶‘ç©4sáJ7Ì0 d’ª÷x³7 Ó£ÓϺÓ»8¬ÄØÍñ¸³tt»»E’ò(—J Œô1{úçØú9¸éópu“xW§’*I&ÎIæ­eš‘¼Eb]ÊY±Uâ¤ÏÎ(G:'ðo¯iuqð‹ßÛ{ð­Õüøq¸xô >ü0\ $ýüìU´‹ñxœ{PÅX7¼°&„’´ØAb”¹E$š9rÉm·Ènÿb‡WE(U}º¬´œÙ织‘SЀ™…$âæ>ÌxY®LˆÑı÷–îR5Í@ 檳Ï)þšu¬dbua ¸«]4ß‚+BŠ¦ª†0`¢³×O¾u#™îµ;"ÈâEàÊ^UÏu:$f&ÙAÛCÚ˜Jø‡šNfÍÏxxB{ÃL÷èp2¥½ñ˜ªÊ7!»š¨ÚmÆQ?0ºH1±“KÃÚa1 ‚2:àP   °¡AßÑã±&çIçzˆ8 hLž MžÁzÕ”ç­s‚ÑÇù\d-áXÃaÈh!ÄdÌ|° Ž8ÚÁÈ·ï}ÑÉœ«§¯œ¢º.Ø…sÃèÓªLŠ ê5D…P)| ¼©¨®dhiÀRâgk¾N̼©Lo Ú'YÐÖà)].qmÉ•UÕ …¾uæ:Þ¼m»¼£'a¹wJ/Yž 9<<\…ŸTÞ×Ñ•o\î†üwŒ§Óš– ºV4ÚÓxLÃɨÍ|hVõœLI€‘27¼d²WX Ÿ+úq+aØ|¬@ÖÉ6•mú>+ Tý>îa‘ë;É[*ž"¢_Ósyc-Ú‡Føÿúm¿ÉXYŔХ¾œ!ó²eÝø^Æ9ÁkbÄy a9ˆÕ`@ï¥Wä†÷Ùnð&9”‡“UF“¨u˜#T®*ÝÑȉò¯Å<Øf\®2v|l~æ¥uçWÞ»çF)ÌäŒhYcrŽkHJMì}hÁQ‚P1€jE9DElZ1‡qÔÐ|3çZ|Š)¾¢°Ò Y]iå†?N†Ã×îºý;Ö«ùÙêøÁ¿Z>ø—ÿ2ÌαBüWEbŒ§±1dv²“¼‚òý'ù'É_@œœÃr^I:&­Â“Xy­Ï †0š…‰M'ŽÜ8UOˆD)ãÓyâ²9Š+5d#©Q{u$JKă±c ß~X×'!\Ï€Ná{MX 쪀¤ãþ¬1¨= I¡ùKp5Èç%0¤Ò’Î$gP°Ø 'kVªÑ>¢Z×((Ž#¹qîŸG_ é+$8o½÷§ög¶Gv²“/‡üBþù/¦KDÿ€%ÿ,‘7¾È§H:¦ó§4ôiOô>±_¾ˆÙ£™Ö¢aË(q“Ä·Ma7Ô„Ý´lÃìa…„—OÎÂ"†qTï±ñ|÷õ¤*“ìüŒqõfc´:‚xöåš“$pA0èªA‚Æ„ûÂV´#æ~ˆ¦]1Sþ5†ˆo ºÃâM¢Sî[®#û.ßß–4•œ&>ÀGpHö¯—Ü N¦ "|hîï –æÝP§IÈ÷àösN¶é`€ù'm]ΓÖ‰ñp‘ªª,t|~N£á ) |8Ù£ƒé>GC)× Ï}ÚЊB·C¿%Ð惼1¥ŽX€S¸Ÿ;œŸQçd;DÀ)I ÿš™g CèKý8}Î!ð)éj€PØNNè<` BË"k<TÒ¼>`ÌY`w(p"pX«š¼Qßk¾@¾¿—!É%y~4Ž<Ò{›*[2Øùfz *))1ሀëèÄ°0µ LÛ)´¼I‚ýZˉ^ûç_Œœ›†ç†¾—ŽÐ~Ï ’˜™anəۼ8gYJ1º‡ÿËÿöÏ—ô’å¹À‘”-öŸ=ý¿ÎÜx0³vºn˜XÈÄÅ…Hu¨©^]Ðü|N~XÑþíý&ìF¥§¦7@Vh€ eÊØ?Íý*7HFüåüe¹/.2úŒñ¾d«%ðQž×—‡$Æœ«/rµùJß\|ðEu° *Ȥ: : u)â8Yê!˜í-šüä~SŸõõÕuÂg6·é»u Vßñ-øàBcüSh6¦y½ аÐÀU­g¨LJåti²ßAæí¨ïC„øßr£ÂEÆ*E9äµDÙª¬ë¯M²ŽáDW¡¹áýè—Fãߎ¾·ç«×Óg‡ƒá{¯×ƒÓYˆŸÕ˺Z~üÙju¾ué)áX‘‹Cã¹)öÍ™ãÄ› ¼˜ÓX6#fK~‚-¯lÔJóÞèxRÃ4ÞõbN±^% ]M±Ž_%Ö(„±ªª/Ô˜éAw²“lý¯çŸ¿CD³ ü£ ˜üAP~n’ţÿßß;Óps%Ï:LÓ Q#¦ý›ÚD­ƒ6?IH¹IDîÂSLùI$ÖÝ «@­~c÷¾«Ç²® 21˜ê®†žÓïùÞM^®IßK™¸d0>º²QÄF¢+Ã9òuxA§£ƒÂ^üÊäF ƒ%’Ï…ó›60€Ã ‚bž@w7€•oŽgàGª_q»#„pu¢1û ƒ]0¦Îu€'Ì…à 7  zÂ2¨Ñø™ˆ!P5ÚïkZÍkš-ôô쌪'Oèöþ”nÜ éÞ^âÝÌ>Ÿ«(R7)ªeEˆå´bpŸ9$$è$•çîÕ19CKm¿¦ ØøzƆ…ås\’~¹hØ"µFĆäLòÉص@ÌÄü€Èžâ[J¨KT ‚@ Œ%×æçó¤!Ô×ÍÓÉ»/ú¡&sö9¡®ã$Åòl¬\ZRì;ìB\3r;<ÿÁÕsŠ^çà¿è¢Ìéä‰÷)·Èk5¹l’ûœw¯-ÇÛNød¤ä©lHÛ ,–D“Ó÷œíZâ$gT²…B0vRZjŽÿäAù /^ž?“ù²žÅ rÏó S‰:’Àfj|3r[§˜²´`Œid ÑéKŠËÁ" ÷é„™Àñ•Û»<§Gu™²l/õÙ}žö¾¤­›Âx6µ¥:Ì]å¢RAcgáÑõ@éQ,&£ƒ]¶g‰¡DYí… :+CfŽpéÝRúÀ"”Ë@¡uì>ÆÎe!4—…CåiYË{dQ/hVÂ$úAÃ,Á ®¡s3jŸ²‚d™"øVT7‹¼¨„XÇ:Æ> ¤()ÿÝ$›æ. 3E¶)ÍûÁ`póÇ{ß»Y Ñ»†äÜ<—q¸W nO+ºus0üÎÛÃúø<ÔŸ=X,ü“°¼ÿ¤®/-}ë\8¯ ¸m¿âZU‘Ž0 v™&¢Âw…«ÚÈú”HëZ @·uìë8¦UjꦰœS"Ê´ÊV½ìÄ´½BrU ­”ÂÙx<Þ;ÙÉ—_’ZüïäŸÿžˆžåä®ÿ &öóxÂåÙr6Ñ©ƒ¤¦e8¥Ë(ƒj|\T}%QÂG® ¹qóHéŠ)ì&aÕŽKcš°Œ"GÞkÐûÊF³$O$ÝS0T€Ï7¹FŠrÀ¬°6Û€ó\9¢ØËÔ3n=Û¾`°±Æ@NÈeuÕhw¦,)ïqÈãpÒ“° ñ5™iR™”Ã!ªœd’m€à¢”(nNÜ,mY`F¸´¾yQqSÇÛísBW µå^p4'÷ ŒÓˆz¯Œ£d~ÏLíýA° 8 yÉó(1é“#îó£#zprB7&{tçð€nìÐh8TƇ ‡RÆ+'6?®¥"˜+¥CÏÎÛÎ9ÀDÑ°dÖ9ñw[qQ›Å>8«¹pb×,[$•æå@¿{Py:…nÇZ¯­óÝ kCX1Ãç“<ÔGR'³&HBbìêQ†{™ <^5êí`ÛIÚâÎ\ÔΕYÆ ÀxdìD€F¡A rRÊÝt7R¸Y“«Ò z×HN×”|Òã8åµ&@U`]¥öùÒf1W¶óOm£¶œ§GÞû—®c>8òƒü ¼ó«œüÚ¿ÿ×O]^¥)Î(©ká&£rðr±‡<ÿÝ8Ur/Ù΄÷ ¦Ç—â~Ï œ«ÖTЛ’¬–ßSapo µaÙÔ†uÇú’Ë>&ç9ÆÆ)+N*§¬š¬^Ts*£³7ö†¢åªè|óZoŒô}}[~· Ù&ÿK)ëÂ¥ê:[ÔM~œ_mç¥D[égî\’Œª$iÙ$šƒBæ¤l°„›1ÃÐ0›G0¶ J*b7!æø5Ì<ûû¼·e‹x?øh8~ÿ­Ñè¯ýà ÊsÉ*´ÍýSá¾á¸ªîŽ|u÷æ`øK„úÁ“Uý£—óŸ|R×'óØvã]lª~‘·au¸á  QR‚M_de2‰(¼¼_z2×ãL)Î-7Å`*ª—KZÍŠôdž¶ ExÔ« uçWŸ¬é‚sîl2™ìÀ‘ìä«'·ˆè?Ê?Iþý7,y)’tÌ_ýö;'ëß¼s*뮶y­÷êåmŒÂuJ¯a¯Â{`£-Á>c"wHìrÉÕNö Œ™—}ôXÏ;âh¨0ミ ;„ž Êk«ÑƒAÄבöÀ=Ø–÷âŒA³™Í ‡î{r2<áßÕiFæ|#òß^LHlûi2̬°ÁïtWmÛ¡ŒfÔpš|öû)oȘzhxy@ æ@ºf-¥RµRô\~´¬…´R Íd 4@N@æQ”gŽùžÉ˜|v~FÏÎÎioô„îÒ7h’’¸úʆ˃YÆ«#E˜ PÁ“Šï.¶ul,£p/€÷Þ]Ëö²ÌÍÂù‡C_ènÔ„µ× 0’tÍ’TöH7& F= >Ë÷ÇÊK„ÉKÁv$M+ߟBw—kÀsØuBC¼Hž[Óüÿì½Ù¶%Irfæq¦;æTYUUÝ5h 4 €@.¾p‰_ ÏÐW BzРŸõ¢a"(RÔHÝÕCUuuUVŽwJ †ÔìüìªT´›¾¿éºMeÓw·i˜îHˆmj% ¯Š°‰jÏö;¢œw]G&Nð m*–zÌí é“ÓºÜÀŸ·×1D¶Ý§~Ö‹0HêxÈ.ç’Ý!?Il’e !7›Ä¿'p'øw#u1'|â®0èÖ‘.¿þ{“²)Ì ÛÇÝMÙ"û!Œ„æÖ·§{¿wЌߦFg40½>½€¹,4š5ãBóûãñï|»ë~òóÕò‡?n—_žÅ¸ÔÚ†˜¹õ"˸%Wˆ?‚1 Š(5Yô÷ä®i¨³²ÕÌ(ÚÙw6)ÿ2¦?Y,ÜØÒÛÇnÉôUi_qÙ‚Ü©€¹6Æ8?::Ú#»²+ÿðÊsH ü¿½.æH ™‹ðEÐt”llJg' Î <¸à¡ø6$YÀ5$!9d¢s¢¤έƒª=‚çµ {‚9ëõR©x4ðÑ9§|7:½Þ?;TNÍ—žíÀ°§á~A€Ì–*…®ZìF›6^nSk6ph0A”Üê!0!ÜNìï’BhØ÷TÕ±3f1‚¡6Öž¸w“ý&0…Ω´Ë§ïøYºGÃ`¬9ÝEÖ¡ý%‡#™–I¾µ• ¬}ÇH¥¥Öƒ;ÈÆ6‡yÓ¸©u#]¬Vtùô }qòœnííÑÇ·ú´Àãñ¸Ø]{©ÀÃÂöÊC!aá'?%è¡DÁï䀊!­Ÿo³›òó¡þɾ\,}ä€ù!pm¨}:’µ~€ÚÝ•AfÃÐN‘´1§¤B”U˜Ií¡ãÑZ›ÝÆtà@M”+ðÈcvdÃsxÅÊo³öÌG²&H—’¨³7W»É—ÆÂÏ0æÔ9Ö©ÑÏ Ä÷‰bÊF3Él¸œ|@Cb4æG` à;Ú«ƒ£_óÄX#ÊÀŽùƒ;ªÄ¸ðDV8IœwŸ|íìë¯ Ž¬$\ð@·L§Ú¬¶Gõ"3<þ …µŒzÓÅÜH ÕP JèH–?[qñIî0îÁh¢xåMB ¶…slsþñ^xM÷ª¯Cg•…V!¦ÞUìІÃðo5ÿ½º¬ËŽ{›¹Ût€²± ¢×8Ÿ¯_ÅÙöPå”oúlSÔýT·í&0K¯­ûdsß4äËÌ:$áL o¥VZZ­†›Q“#iš<¾Ëö, ½g¦˜®ŽiyúI7¢fvF£½'š Wîb»mŽi]ÇmEÛ¦§Â5ÍZ[£àêMØ" ùíéìƒÍä÷¦M¸oû3UÖÌá"z·`Ð0ÝâðÝ[Íè×? ÙÃç«ö'?]-~òIìR”x0uŒãfªF¤nDØÈj8†ªýñ$Âêö—γÂ؆°@ʆÛIbìÏ¢Ý(Çk{?÷m-«–êC‚¿çò"àšŸ ­ã"²Z­VçI'àOÿôO‘^qWveW^¼$Í‘ÿ`Èÿóº5GR‘†.„â³Ä ùX÷q¯qF²ø{¾Ð7LvfvõÓïSðçD(ñ ÜšE.Åt`Ø>^„á ZR:ìAÔj*Ãv P…½’J§ªG€Ž9P×{ðÄ-rmt–Š¶ ¡?Š{çkcf_tÊØÐL>â÷KŸ­V4vŒ;žn2x¸R4CÁî :. Þ†øœökÈ »>ˆ½‚=-L‰5xñóÞØÚ‘©rL+ÁÏZ³†í}ë ªt±;Ü«®£ÇggýŸƒéŒnïïÓ£#Ú›ÍvO€Qªì…zB¹=„ÊúlcæÛ¨*°·ã$éVY΀ôá3Ëœ‰¦®d ýîÈ$ø›€´N(×ý£cHKŸ}ø]¨ì& ×Êã…œ1ìÆÚW-ˀ膢Ãtá9|Dc+éûÁ¼Èÿ‹v(lté˜Ó9¯¬®ž:ð–J’(Óg(Á‡¼ãÃýHïJÙSmíÕuÛ¯¡¯S¦JÆqÇ(´L4±}zÿþý_|p$ð*é<ãÀ‰-Þ{[‚_pƒÍ+ê³Ø$TiðÙÖÀuƒ\?*phÂ(ìÓtÔÐrÕ£ã~U̦Ï69ß›B4êßÝôómàMúL¸ë¸>Å`±ÁG¶‘’Ñ똳噲§í^žz8Š‰qk”cżúþl¤™Ž¯bjlÓR¹N\õºp™šYs]“mÌoÍä}S \náË?ãI¹#¦‘"-5V4jF½.IJ,Tq>dàŽR=¦ÔÎïÑòäÉrÚ¯íùµã7ˆg4š=‰ÜœÍçýMwÚ.Š“ͧ÷ÅÏ›ô\Òÿ0² ¹ [dBónÓÜþp2ûî­0ú•&ðž£6ѵÉV9')iñ}ȾÀ'3jÞLšoL&ÿô·£<~Ú.~úyX¬w‚JÈÝÔÍ ×4 ¥³©*èi¥~j¬5Å @”Öµ8M¾·ìšbÌP&îr»ZÖ´Žª IDATº|ÕßoÙ–u¸v ÒŽF£Ë?ú£?’ï}ï{߯wsÄgWvåï·ü÷Dô?þôAÜ­æ…³¿ŽÂÎú8Q$6 P±–s±~ûV[jMjBØ å´æ#IÊTv !˜$óVÐ_6|òÃ@H¦‡'9qj㾄ν€e®·±V*º¢ 9د×lx·ÂñS¬$X=õá 鞌a¤˜>Ô®V ÊݸŽ“÷Op¸õ$û'B¨ ×ÀH"ì7ÁDê }?… rŠTd!By1¨o“(Æ,€*˺ã­ÇöN,í?9·!’†<dvJ¥ÅTš½H-ž9üt¾˜ÓÙ|NŸ?{Jwû°›”éfû\V§Ï¥=_H7ÈrÜ4<ÛHë­cùª1:enÞiF÷?˜L¿}§i>q8”[@äyðq㨀¥u4=y€úºŸÖ̽‚‰ƒˆ;‚§`˜àC £êíbEÃóF„Ì ®Œß¹—oÖõD‰óe’k½QGý–kÆØŧŸ~º`®M²]Ù•]ù)ŸCjÞˆ|ü‹Þ1rÉŸ5màÞ¬·RÛzî’—L wðpïÎö0'(dÆ÷^$šqŸ¸O<’Õ¬Ï`R9 ±õÇfÚlVùž%¶g”!^M1TˆÕ3Hàý$ƒ¥'1:k5ÐÃõOÔ±ÓÚ÷o2‹BC´cÉv!ÝãÁÊ ° ¶Aî$è–Ïð%®neaHä@Œ¿{±×ë¦ïéZÃß;¨bƒhŸ!3hJ[à ‘9àh[;0R|Ï2Á`ø­‘êtryI§ó9MNNè`2¡Ãý}:Ü›Ñl2¥ñhä÷()HvÿâÆTÖaÜi£& mêúóÛòﻞ¼¢UÛ‚­ì¶”Ž(oq ö±æMº^õ'…}LDVõ¾;1ÿÏþ…0­ÒðËã[Ó.¨©Û¬ xœµ´èàÏ,Ú¨dö J¶‡Í ñ:kûõuMrw…ä8R¬Kƒ¤ 1;^}È^j²¦ª_»–,4Ìź¸¶Ú@cxB>^훑? áìuؘ_¹3¿ÓJŸÉ8$ŽÔž£ð°ÀnÀ³V±CbûØO¿N÷– A'¬¼¶ü^ßÌM˜…qw DËu¿Ç²Iȵ¾ÏMËU!5Ô+b´«‰á³òEW1‚ “°j®kæÉÊ Ð²MHUááÿ´>¦‡Ô)ãC×1B^¶íh +âEØ*aE‹ð„þnÞ.*¾Õ…7e/¼ÑÌFoò8Üé1VØôÙrÓ,»o¿.Z%Päòuç÷ˆÚi¿h7¿ÓÆ®{Æ£f%"{„ë1†CÈ ¯ßù—ëû­Û˜~Û JÌ„úÄôi…[¤tÀ$in¯€Ì‚ü¨‚ñìʵ=@ñ½JÔΫ2Õ yÉHàõ;úûæÛ¨ž‚@ÊËÖÚÑZ9ש<ÍõMÒN²µ-ñpƒ4°ßÔœ1ÖšCÝ+§Ûß™¥´áÔT`lj·-ü£ìÀ*P‡ƒ; ö˜”·ÉÆ5Ödž ™THN¸­mþ?œ•i;)ÛÆ€#.C‡’vÇbµ¢§ç´?™ÐÞdJ‡{{´?öÙn& ( l4Ÿ¡-ÑÆ„¶2=Hp{ὕ}bÝUáfØ—)|Õv=“º`‹@» ø+8xõeÛ3vj€ÅÆ¡f\/ÅìTïÓÄ°ì6žö[Á*}|±aÎG!Ò#ÞŸ¿­oÀN[!*‹í¦c<û"9ŠDwRº^Õa8¿C È[ ÆŠµFm³cjhuñ=i‹ß/ûô…ô1Èãÿå?|öZB=¿28òï|§{¾<{Fã93 Šeñ¦A—@Ì *Èɉl»âKHÓRpJ„ê!¯ßo”§«k´'nsÛä¦e“¦Æ&VKyßV“dkCž0žrW_Ù×~¤*á:§í²¡®6§1õPEášüŽF×Ce;áûÞ4;ÍMîy;ä¦?Ç%^tóxçtÒ<ŠSšÆñò(æmž…4 ‡rXê6à\ߘ ÅÕZ<Úó;“®H7¿rNQ—Â7R#6z ƒ3À}ƒR¥ÇLánst÷NsxüÁhÕžÄåéã¸xúe{yò.?‰«§K¢"d¨˜GBC/D…(‡Ï//Þš¯FÍþhLûã1Íš¦×/q_ª7`X õ ´—T‹~žÏ ÀH2êº&Âx/·I{þ†5£Ü‰7´qa%qŽ™,7»À•Qíû¨]×ǣdž±Y§­Jl%^#æò X®Ó!ɆS‚|Îf³Ù/ÝûíÊ®ü*KQMþï¤ôËüzÉÆ\.~ø|:šÌ‰ù7øi.Œ@¸²×ÒìjqŸÇâЕÀÿÇç%îJ:qÝD=PÂ$ &^$ ­t«Ö¼²b?(Ee™† (ó@*@TÏ¢<Ó¶{£>¯ƒùšCm <5¶÷…0§Ê—"’ Úsm”¢U*Wƒ*\Ô2kõ{¹S‰)’Ñ¥.ÚÅ\YþþØ%L@VÏtj/v êã•—Ë%],—}ZàI3¢éxLÇûûtëà€ö¦ƒ+:×Àºs]ö 5÷¿*,ýÕëŠ$`$v¦=A6ý>¶¡ú4\€yU=Ï@uÚ5$­KõøàbœŠËg•³S¬ELCĪó±rî-Ì» °ÇC~idx“ ï©:5ÎJ+ Îë^¯'q¼ïÑaLñ9tFŸãñnT9ÛÎêÇþQ)ÕÐ'£Ø¨çýLRŒ;$ÝCÈ;‰t¹H²òè/þâã%½†ò•Á‘ïÿûÝoþëßyòÁïpa›‚ŠÆœÂ×ÞÞ ž£§úê/Ó´µktûõî׎©'lí”%W‡Ça6„`Ôå&á´]rÕµW‰¸âÏ›²µàu"mRùX®i²P¹¡ù¢…Ž{¹ÈÙ Ù¼¯Z·JWjñÀ êwòÑÆ°šëB]è†`Ʀ{nêÃm‚¬Wõ÷UÙ„RéR~±–.:¢‹x&¸‘†Ãæv¸3y/†÷eî0óH±÷”´RÃï‘œ¿G‹‹ƒ ŠØhíÛ0A]—œÿA\© )=ö wQ,ɶ˜ÈnjÿOšYOïÉþ½÷Ç·—²œ?içÏ?mO>ÿD柟J7^/¹Q͆~aîºI)º¶¥Ë®¥§‹9íFt8žÐÁhÔgë ¬êÔëÎtq"´mòÿ @© úðí#5 ¨<$\Ô™Šõ ½é«ÐKõ;Ü$FˆH.˜h°ÿ¡‘–OǺØÐ*6þ^¾t1J·føúÊ„9¼3¶‘º/b{¹Ü€ŽnAxíÄpý;‰æØuÝåh4ªê]Ù•]ùúJšÇ `È¿ýEQ}•%Ù˜ÿúŸýÊ“öÛ·.”õª`t¡³•Ù° t¸á-¥GªOÅÕ~ª××N_yOQÇ8 îbÊr“œj“—.3DUÝÍ,«?ùCEÃ> "øN~Oõ% øý_Ÿñz¼Q×Ô‡(ºm:ãëKRƒëæ¤a bο9A^'MÖ0dÑÔÆú"l©ò·l%¶ö-^¯Œ 1làÁ„ä¡A|“¼íd÷jÈŽŠõ"ÀJ›dÎÙ}":üŸñ¤©Y ±A®ì{Ðv‰‘.ã’.WK:™_ÒÃçÏéxo/ ¹zZ`?È“˾_÷¿ oKاOË›Þ[d8ƒ1…Z0âÌÔÔ€ÑY0Ü0sÑì<ÿ·°Ï*0M40\&—“ãÉæØö«|ŠÇ>«œ¬Œ¨S)Åé60¶hCy1`ÌÊ]eÏ$„pÒïÇDr;ßâY ƒ±e©ÊsW× )ÛÒ뎭䭡¡çú> ^­©„óÓ5Ÿ†gÄ<çÔƇ¾ê8ð³Ñ(¼–°ô¯ ŽP¯Õ°:!=št!KGâÚØT,ª Ô!Ílc¦«3а£hÅ"Ž+žÝºE&”4afõ­œì:´ 3nŠseûÜ0<‡6€$Å5$]ˆÝ¼ cÉ™“‹E |ÓUÄÛÁÁñ˜ÕÊ)OJˆ{Ä0?÷,Ë=¬Ÿ8‘9ŒÆ7yŸœè àä*Ðåªöûªz/Òv´ÔÅUü‚Ÿµ_6‡ãÿJwšÍÑè=ž6ïËž;‘ zƒúMjúäá#_¨v¹êP™F,}ÐÊ8EÌ°€§d WéÃÚš½½0žÝ{·Þßzç´?ÿiûüóºç?ø²[]t/ Š`I¸MòuÛIÉÙjEç«U¿ Çt˜(ͨÏØC6ÿª´ÛR‰±[‰òâž@˜@ÓFa)6&¶¿Á`ªzŽ®´WWG º¸ëb-YŒTg$–ë 1âÆê„–gyrbÛðWVzárÂä·§ûï¿;›þN 0:i—?\.òãÕâáã®[ºñ±E¤ ׉ $kc×uç¯#ÿü®ìÊ?òò€!I?äÑ?ôæX­èDH.̨€À©Çpáü‹ŠÙ)Å)(å{lO!èiªÿÞ÷!rsøJŸ]±ÏÓDâ)¥tÀ©¦çœA’¨ÐúEpà…`þ±~à‚:K=H`á5~Og6øÞÇ9|´‘,Ìgw¦XJ€D¯ÃTºX_FY­]Ôì²­ 1\ó¼TáRß@s8(¹¼ÏL“B ¨ÒkÁæ”5ØËœH1­†¶Ö6S¡ÏÞƶ>7À¡Ž9Ø@™m-.¸«Ÿ°]RˆS—ÃzEßmw«¹‡-Æ—’ÖÄ6‚UœÖÛ.¥ËÇ%ÍWKz|vJG³ïÐ탚%6IÜÆ4©ƒYr¿Š+;KqI¡åj9èŠDX¸š’™vmó•å¤5 `‚- pŽèφ•Ba„òôò~ôõ„Œ3§}Îyà‹-ÃS£ shmÌ)Ø¡íhsX=(U¢•<‘sð¤°K•N€í±OÑ+0ù}.ãàëcfÔýcKµ x`x>DPb^Ï<‹#Y¿v6®Ùð*Ø‚FÄ-IûhüZÒÉ¿pD„ÎH䤗MˆC„W†ÆhΠ °›Y ^^Êb,z­Æëˆì^²É*9çðÍkF¤NL¯A®cyÐu Æ ÀM²°l+œdˆ)EvæoN]GèÓ݆ Átq1GÃ|GwC„ÀáMþüá15°t“Ì2Wµ×¶P§Mÿ¿ê»× µnuac¿Mvíóås:¥“Ø„Ò­ÑíænóM>ÿJӾф_mú1íçär°pq¹°(s^1W2K‰=‹dÔ Œ’ô‡±«"±i–“8pÃwLJoÜmöîþFwçƒÏ—'Ø>ýá'ñâÅcöD„9òˆ!±4IoD<[,èÙrI“z6ÉÑdL³Ñ˜ô=‡1‡q`Cc¤“‘”Û¿3POÖžU0F|?(6Oßp ,ÚÚávŽAACj` â¼sÛJ;"l¯=µ°ã^Q°¡NiõIøT=k¿¾’€‘±wôOîO&ÿ4ôé–‰îM¦wïŒ'¿ñ~Ü{ülÕ~ôñrññ§Ýêì¼ëÚÑ‚`H-š×…V³Ùìµ½Û®ìÊ?’òDTÓ¿?þÇÖñÂñŒˆNdµa±Æ¬"N!‹Ô£Á½I„ N"°OÉY$jcÆ|¯ ¬‚´ ß"âƒç¢3!Z=;ª“Í°`b`€9‘ùU ÙÁ³Ì î‡9Ì€Û7 «BL ÔD×y*þone ´=ñ¾Y²ÑZ-hzàÀ>â!\l©l¬¤?rr‚5„¢Ê41Ðê/ÕQFûÔCwPèsÈž£À€1ä/2 ØÁ”ubážMSšœÅdÈ<‹ßé{õ‰"8×ìüŸBg{ËÚ½©¥í"=;?§ç——ô³ÇéÎþ>Ý9ÒO'cKäà¾oð5ÐNàž%’Róö¡Ú\gP`CX_Â|ôØ ô³É©ó”mÔ¦]­Ï/ìó 2ÙŠúE˜‡è¢F›w>f´Þ˜)Ù*ö„ü³ê·íàkJÇU0`ÉS'¼åPýΟ>IöëíHt? IË"utWÎàË,šêz ý¤`“6´-Iö®¼¶F`g+$?¯&!;pk[ë åpfÔÌÆ¿<Ìj ‰œ²fJo4moíü8Ê„u‹Ö4GðtÝ'g…ÄÛÇ¥¤Ö~€LiC“^—¦w›&Éh4²ïl ‰ÙöÝmiQ¹Î‘OÐf Yn‚ÍœY@yÂ@{±£öÎÄ(Li&¹¿è9´õZ,¹röúg £qó‚ZÛtE¶ Ûnº÷U€Ëuß½é5×–˜°¼¸¢GË/ã#úròæ‡?}ã7ÿe3½û pST¤¦Š8E®Œ÷õ]Ž¡/Ú–¬†E#$ã†{ÀľelHU‚E¹Oä=šì0¾÷î{qrøŸå'ÿ-Ë‹gL FT >±§;º¿ˆ-—ô,‡ÝÜšL{%õ1$6ÚÄPH±©ª-²I²T²±ãXka/¬Ž;¹¡çãzӳŌ­r°)oûf÷&2¦‰ÿ<Ü3Jdå5L’Z.µ-mZˆ®h——)‘ðíÉìίíïÿþa3ú Ï/lg(½ðì`4~g¿¿óötvñOºÕOWí²˜úYìÎ/)Þ0¥cæù‡~ÿüÏÿü+×WvåqI@Àÿ€Èÿ‡ËÌ?ÆÒŽä$F9 ¡Ä§‘ÕA`+õZ¢¸!U¤:‚¾ŸÁî NOIÉÇÅ(½&­¸Ç‘h?§>#Šó¼OhMõ´û#dé4l8a5 d°ÛÔ‹–…™b{‚:¢%iNO?Ñg+óBõÈÐKóêmšC Í/:Ÿ wšÛÚK΢­!ÛKQïÁÐù¯Ø5AÀ*°ìrlÌÊá9ÒP8¶z‚Ÿ³ÂäKÔѶT{+à5ÐVŒØQ$+»4ƒ%©½òêrë C‹¢9ìœÙßzè¬@ˆö›…$>d¬Qû²w<Ηçgôèì”ö&ºwxDwŽio¶7„väÚÚFž²Ð,—Ë\5=$8[ú&bØX”eÙAkÍÞ…\ä,@ó…bÑð.®tˆ v•Ž_ÊïVCM¬ÁÒú³¥þZjÿ+ÈH:•%RéÀ è¡ ¢×ZF›=…ѼÑõÙhï~h`ð(;ðÿñãOÐk.ŸúÝlDl ²fJÿÙFåuå¿hL ù¼RÀ$+6´<99¹Õ}J_wy%àȸÂ|–f17!ôh’šn¨õJ‰”–c.Ú¦†á ¤#™:‘{DL$e’ZI“WŽ\nâ±M‚`Æ&g¿ @Áç¤ÏfÍø|Ùë o ˜“䈴ý8‹T4€Q´ŒNæ “Æé’RÇDhmÚ+(’gsZšÓX±/IÃj¶µéMµB궫?£- ÉU` ~¯îךåòBer<š½ýÏß›Þ{ð{Ídö–/‚ˆì¯³GlÐv…UŸ9Ÿ²¬ºõO›Ö¸ =Xf…݇óiHg ^b4tÔ­–ºnÚkèv/Ž$²^©’Q†úõy§û°›¶¥ébNÇ=›dB³q9d¤F<ŒFpG€^—ÛRŠP;8>@L¢æ¸y•h’/þC(“/îÌn¨hîvm®Ÿ™þídÚ“ Kzrèsq%‚ë•mü €‘ÃF¿;Ý{÷ÁtöÓ¦yÓ᫲hÛæýýÑ䃽Ñä½·f³Ó‹võ³Ï‹~°ZüüiŒKª@›òàÖbþ———ó?øƒ?ïÿû_ùveWþ•´ïÿ;H¯ûóg»’KCá‚Cb(Jgx S;1ù…8$xØ…ÚGz€ Ž ~'ÂŒlY/œÝá@篽¥•@’ÃH´Çƒ$ñ6/ú“ )ì_8Y¢Ì1aSw tuéû§9ì5 >mgÁÃrÔ‰EÌ µÇÕQ*6Z`q(‡CÂ*ô8´=Åv,Z_J€B·`ÊaÃiº³, †‡¤û –‰;ÜÃOö¬L;HïÒé½s¦;{ßÜßÈp eŒš¨ÙRRì³²@Û¤F2°¢Î?šØ6‚¼îßÙ øÎp'`cömÊT0”y•D\/Ÿ>¡/NžÓíý}ºw|‹Žöh4jÀ ö0ä”}&…Ñ´mêë™{x·zmNRiK€ ê3m c\`…!qP €ÅüDó˜5¥9±Ûn¸¡¯En{›Å…Ñï0µÏœ•¯1³½èïÜðû{7ûLr?R<–þ(¬ö•¦©&DôB†v Ù7 Yk õRë¨0{=½i¤hç4H'G’à:,™µÝ×% - © ¸œ­æ£“û÷ïÿ1GèŒYž…ÐDé²×Bv¶ó5¬„dÚ—8ÙÍB®ÊT”mÃA²õXÿ«X9GÜë—®bÛ=Z=\ü—öã‹Óêf§¡›BBêrÓ *x¯Me6¦Ñ‡wã7Þ¾G¿v1çÇŸ<’Ÿ~qO%R Ál€|ŠÅãhuðÀ¹L0è‹g5¢Ê¾ z¬eéäóA–ÐŒ±ŽD›Y"›B\^´¶µå¶0¨Zë¤ê~Úv¿«J8øÕ;{ïÿá?ïí¿ËM3µv©Ï WQXHÔiw¬Õžf—XûK–EÓ…I’ÂntƒÏø_—_!Ä”F­]e€QºÐçƒ~ñ"Q‚í#:Žìô ðxYý{Þµ´è:z²˜÷â­GãIŸr.µ/ò YuZÑ–Ì'U6=Æù.Õ¸³Á5Œ‹½]º®ª4šÀõl¼Ðu®¥un NFf€¢A!mÛJ÷RM³±˜Bf˜gÿÍþÑoß›L~7p˜@“ŽHí68Ó[zõ®f IDATŒ9ܺ5™Þ:O~åÃîàñÉjõñ–—?ùq×>KÈŽ¤lNðL¡ãêøøxçÐíÊ®¼xùãügW¶”ƒÉ2…n?ãtB—0#{¨¨a­Ž7‚Îàà—+8Ãz¹&R)î|ª†ÖÁ %Ïÿ5 %UhŸ†4š‡‘ä¶iØèû¸?iS_¢ eøX8šƒ9e·£èSa¦tL*;I×&Áv¥²ý£ÏàAc\èìA7Å_ήÓ)9à­AxU&î;èúe tí>\Ϧ"ìEÛ5*(¢@ûk ß¢v´Ð²¤‚Êö} EÌP“çN~}a@ñ9-ŸþÖoýË×ÂN~%àÈ}º?§Ï–Ž&÷‡èF<)Ÿ)4¼®jCê^í§+Ò—ObÇÞN)²» ÷o›ZæÕŸE|ºz²ühõùüGÝÃù‰Öñ:Ç÷*ÇZ˵(¶<3ýɈ¹õ­»üá½#úõñ˜nÇš†Žöè;{oÇ_yç.ÿüÑ)}ôéSþìä’Η)_Š$j>¦Âà(¡ ¼Sdbß%©Eüô<_ëëÝzì »ñî¤ÇÐŒ…C-\ÕW‰×ÒÐä*Ý’MÀÆUà̶М¯š;$JºóçÔ^œROi²wDÓÃcÏö)4£A©Ê°ø®õŸ.¶ ¬Ïyu@‰7ëŠtÃB4íû7öQ3±] 讞ÚP2Òõèï zè)G ±"\œZiÛœ0ìòÇWc©?¡èºÈQÄé¢N­hNB”(‘ð 0Q¶G¾6]Ñ‚f»"zÐ]1è†5HÛ1uc,¡«UdÅ“Ž¡ÊÝ*Ê‹¶¼·×¦°–TŽšfúkãɃg¿»×4oZå>®JH-S*™a>{íÆY6øxD|0jšƒýÑè[oO÷Î'¶O¯ÚO>^->»ç‹—+‘ËO?ýô‚yŸuWveWvå+”ûDËŽVç,MésÔ`±?k.FyöMq¥H±– ?éW¹ç MÝùü=†“UÉû–“E‡f2d–à> p NÇ_-8Ôø^øov‹ps­£þ’Øm´=Sãû»–ÖhnáKLµ‘i?#è¡}Cà¢+ø¡§â,RìMfC€S×dà€ìô^{(_§G©ù¾ fÃË÷sÌk¨s“™"½Pl-šßÉÀñ=l,yïùkÑYýï"€ºÏBŸÈ.”Õü@@@ú0ÓIÿÚÁ…1Ó €GE&ó¿¸OÁ4Åf“Vz“¬±6sR k)Ö˜‚ëlþ+QE”ÉÚÃÚ?Š…rQÑ?ÙÿÕ1‘n8 ÷"Ém!âŠnn"»HGL›Æ]¿1¦bõÜìa‹ \á«¡ÁfíWÅÇú1Ôpñß+ñÇ×Í’lƒ$xÏ‚",–±)˜0C›DiWÛîeЃMZ$æðFÓýÆlïÛof¿1|ltÕ–…±)F Øjs"V‘# óÑ^3>z·}óíéôä²í~±Z~öy»úéY.!“Ö®ìÊ®ìÊ+*ßùNwqö£Ç·nS»¶náÚìÍÖ2)ÿµrÖ!ˆ5a#¸  X<8–ŠÔRÎVvîÛzÿŒ…è /WY$á.î)Ê~8<©ÒÊ¢]ú™ºâb–Z3wø!åq~ ƒè(¶-éªg1l@ÈWp;•ÙC‡TÌUèœíƒ‡tîî2¹øªÖ—ô\”]Äí$˜ÅDEdÕvÑI ŸÐä ls•·+BHÔ©o°UÇ!0N¨NÕ ïmí ×G“ÖàØyŠh»ž*͕ܵڟ>l@ØÆTQænÝ)˜6NÀ¡hçbXJÙÖš[:®`³™m§ÓHà+¦‡cF/Ž/ 'ôQ3hTDeM9MY[Æ'£RØ¿‚ãÐƧÏy±öPks¤`^á]ó˜ ƒèj<ŽÄw¸{}u] ¯d£9­òØ %˜ªmª}®=ÌlFßÀ3)Ç©½.SÑb"Ê\€Å%z…z~•Ÿ-Ú”ô%¶4§×T^ 8ò½ï}/þáþáéÁÁÁŠNã“pÆOéT~÷øíÅ·eŸÞŽc¾OýÁv¹c!H7¤ŽÔÆ<6¦¡wQZZÅ'«'óŸ®ÎÚ=jɲ3ýªL4Û>{Õe:¦æÎŒo½}›Þ½wLïíMù­À2#lkŒr¿Ì¯>™MéczëÍ[túôü¬ýä$ð³ùŒ_Žé¼Žð‹ëz$”OÒma— ”j`‡jÀT¡ ³R2|xÿµ}§ZÒéé(‘zOA§¨ 5¡Ž+àð¦%ÊKÁ2A×1¸Àwc‚Å0(†…ÕÛ¡Ô Á1,\Ý[› _ixçJóÞ¡ 5CÜ ¼%ÁÍž¦.Åw}Y#:r×®¤ ™é2 ¡ùÕñä­÷'³ß¼3¿Ï<¤éÕV(­MpŠ¾\‰ÌèW_ ô #æ[Gãpk¿i¾õÎ2üçwxø*^mWveWv¥(ßÿþ÷»ÿî¿ýîÃã[IíÁ2´ƒ6Éêì­<`³}5|S1À 6•ì €Êˆ–O|v¡ [bÞ«!Iº$³f³¢”$Yì%¼8ƒ~°Œ’’:Q|O|í= qM`£¼“µƒ;;®cŸî”ÛÝm?>‹,õ6äM[®n%ù©í›¶XŒ…õðeΨö5²jw]S ‹©ÙFz0¢4LëFTìú}±ï‡ò²õ™5Eq°{Ú5 Ra¨ØMZY7»\¸?WÖM3˜#\ÕWm;f‹©í¶ÒB¨u«ß‰1›±¯ "Xû‰øP9Kù_Ÿþ2uýj›‘K×´ŠðVŽžN¤Jˆ¶:pv’†ß ÛFlÊÚ83@!Q xzGH¥§?ÕËšX…˜Åº©ø.y8Lºf%˜éEŸ±†«C(ÞÆ£ÌbêA=½Qª¾(ÃÑHpÉî ë’tMG4:_Ø×S^æÈàlž7M“ RKX„3¾Œg<âÏhÆG¼/o˼#³ðM ¼Wd lqNý¢Ù¹8£ù.@•ì߶{º|²ü¨{¸ø4>]>î.»Eò _:ãÈÍßóF÷Ÿ ÷Ãí7èý»GôÎlJ÷›ö‡ßnØšpáªP³üú£ñˆî¼yë’î®èr9¦'—3zt¾GϦôh>¢ ÔzóT&(á‚dNdY— 0e Ièxá±Æ6Ðá:m‘«tDndlÑÝFsUÖ ;Âf´ø©N¤vyÑÿYŸPOh4Ù£ñþ!¦{4šÎÁ1]¤ŠcªMÿâ²ÃÍ[¤[‘t]O?¥Q€~¬l£þ–±£øRä…´5kë` 8(«Bç,†¥“•j¸ÁÃÛIÁO#Xm Õ8ÿÍ5HsëGì'+i 7AïYž„—v°ÓKpÄ^|PÆýÊ‹Ó„CóÝÉô7ߟîýúl4zƒ…Æ6tÀ:B*smãæcq¹É©Q[}©ŽMOz6±mÃHºöhÇÙ•]Ù•¯©°Äljáo8£Asƒ'¨c…NŒý—0›ôÕúº±¸•oÿ0nÝT®Ç¾FCèlÊp“Ž¨'Ü ¸r ¦îÃn¸IXt©C(Ö}©êÅͲñÜ@™j—¡ÁŽb¨Sê!Ù2”²mXÊÝC`Fá[Ľ [!÷ €/Ã=-\CÁ)5Т™ Bïb½“@cïÆ6°Ô¦õ(åë6É 3ÙÿNî-»ùû9óÛ5gð@#3Õ8'}° Íú@4vS’U{C'üQþΡzpåâæfËúv˜‰q}xù÷•­-[‚þf ¿b¼Ãpñ³²‚ƒDEÍ Fƒ¶øPÅhÁqÂøü4ôÌŸòëüýóOQP,ZiñíãåSzÞ”|&•˜,†\ûÜ$Í< bƒ ´Å8«/ß#¨˜kÅ ƒ…zó©Zžô?Ø€$)¡ç“ÐDüõ–WŽt]wc\Ô)R¥•ÉžÓs~J?Ž‡ñ>ßʔޣ†Óª›À2¬¤vFrD‰±íwÏVµ??ÿ¤}Ö=£E\Jç_x™¬#Û2¢¼ Ûd$<8¦Ûoß ¿z´'ßÜŸòfž$u½f]ß·Þ€thú`ÚþÏáÞ‚Ü:¥‹å„ŸЧÏ÷éÓ³1-b€I\ ÖŠ¨§þ^¶ï9Ä ‡K Dop˜%‘í7()lʃ¿».«ÍW4®ú^]^–I"®®ºf*¡ŽF/ŒÚ.i5¿ Åù3JI~Æ{‡4Ù?¦Éþ59½mÉ^ÿg]‘UrLû…'=£iŠ-Ç.X…"ÉK9ç!„&Ä”†ž` ÖÃqc_ÉaÜ”†LÕv§ýþÑob÷p JÛíY˜A°5?‡¨ ÷Ækæ&‹È^ßÈò?Qš>¬F7{+I‹ûj…ÎMï½­àµÓ.Œ—«ï^ßåÉ„‡ÔÈxɨ]¨–g£µ Uó˜Q»BéÚ}bâ„õ¤„Í´ì"?™ì²ÕìÊ®ìÊ×T:æGÂ|Ö¯@•÷]‡Òh)„ †¼@X Þ§ © ·ÀŠýÈ#èH3—N·;Ø Ž¶x;µÃÆA¾Ÿé 8¤õÍáïLªB®»’Ÿ‹úPÁŠar!×L]m7Ñmmø›ÿíâÓ‹Oø$žöIzW\ÇT¸.œ#•¶m‹Ô®W ›ÒÁ¦2æŒ/³oÝ¥÷ߺ¾;™ð”@DÒ6^ *9K¨l(œÌ*±´îDŽ4E7-ï]Ò»wFôäìˆ>zrD?9Ó¢CÀÉWm£?¢à%J3 |_M S?Çi%Ùn¬+B\êð›úwuÙ”¢wÓw®KÛ«å¥ÙF<À#¦÷@8Ká©vH­Ûk”œ<¡q »9¼CÓÃ[4šLJ`¬h{Îagu«%„ŸU6a±NUÔÂîåÀŽ11~ÇŽÒK±Qù.‹ïÀxÒq]![®3TI/çá”_8ùƒKìõÝ gëE¶y•àÜêÓŸÁÁL5.{1³XÀøHæÐ2ë¡"¨™ŸHz™\{çM€Ðu`Io8‡0=],ÎWK]:éÖtF“ ›4Biô•kŒ¹%øê‘çRlQz¿Ø¯_=(Ò¶¹Ax5îV'ÍÅÅŽ9²+»²+_K‰‘Ÿ¥t¾Íé ör·eõ(.ÖÒõr_ã L‚õÓN>uOw,4cì,úŒ ‹­±xÊ=† u ÛjÿOÖHÚ$¼O½&I¸`’ éÖ ù>8qàǹˆÂ‡‹Q„±9ÖQ³ÊT§c¨Ÿ†‡0€ž…¥Ï£…88ŸÆT„ŽU]€2 6•Mƒh«¬õ«X»ºúWNg[…¦ø 9Ÿ|¯¾H~~Èr»ÚâŒ÷à²}QdÕê f°¡",hÐpɧùv²_ìx†¼õ¾5?‚ð­ÅPô_k˜ÍfˆÐÅ´Óž·D®ÁbÏòáf>‡€¶‰Ií3¼l‡M«`+øÌ„Yl°\ Vg]€çSæ©ŽaËðT|@8ˆ£ser¦®‚¯ÃÏæ]:ïOZ"|;’¼1°EúÄ%*ôŒd‚Ô¦Ãã¶j°kȾ4} ¦±kì¬z挮y>«ÞÔmLìg2%…îJ¤Í×E¯¶ûA²ŠANšw.Zú+z-å•# ø”Î7çÍD§ÿß»’-­ä¹ü¼9á/›#úëx'¼§ümÉÝÔRQ3©„ ݪ{Ö=½üÛÕÇ—?èžv§Òßpxàu™ejçy4mLçzƒ‡÷§Œ¼ïxüÎmyóÁ}ú­Ã=~I¦&°›ûˆ,Š/ªêDƒ*6€'¾ÂùF22š&B3YÑ7î<¡{‡'tû‹{ô_Ñ*¢¯56ú`º‘JµÜ!zç÷Q3¢¯u…´9oKÓ{U Ý«€Méwëßm ÃÙœlëï¯Tú•·goÞr“6ç\§qv´¼8¥Õå]>û²g’ìݺK£Ù5½6‰o…}MV$×»›ÒCZ§ÁÌ™ êïf%¤ˆ?–!}¿ó× 9d‚õðÖÉéÅè‘ëÀ€í¨ôÊ*®V 4ꌾ‡›œ-Úëá!z Á3Ø$"¯WA£Z3ô§|ÙÈKäš•¨1¡î±K©|‰Ƈ‚%ëíc'—iZ_zà•h%=YDzºXЬièx2¥Û³ÍFã¬:¾ÖúN#F üJ\ UÍ —Æ`èD× ’Å9Ï.ß}·£þðe†×®ìÊ®ìÊ•…yÊ${…ä=¡nÔÓY–ҙǽ²ø8󻸦ˆØ‰{ ³:•‰Qï¾Ô´-Ü0PÇöir—³°AÇ"ý¿’r#GDá"]ÉŠËl¹n P¨Ö‹Jfг“ øO6“îîp*­û(ØBÆl œ|sæ‘áX\‡+ùÇF]dۛʬjy©’—†G @€{¿dÀgq0!*ǃɀlKo#*Ãà•q@„ì¬Æ¬’yôÅè`e%dÖ‹°ÓB8ÆĆÜp¯1çTÅ™}”Û=jÈ Ë:{†ÝD4–”VdH5ÜI]÷Øu €Ø1„PÙÇØ ÊFWÉ€'RV*jø˜ˆúûŒ"™ö$ÃœÒz°Žm¯³3}njš‚€ŽW! ­­sår(¹‹ÜŠwÛÀa ƒÚ‘õ^D} “{Bñ^ÀÐþPÝû„píbÍ$ƒ gÿŽ5 ã`Ûf M}Ô,b&cV$…È+¾ d¯"òÌ}cFKõ(Y|™,­³J\£:‡Öý&!Z̗˳ËËw;¢×cc¾JÍ‘3Ï8b â!.ýB*ÒÒ = 'Ý#Þ—ÐÝæA»Oße¢{1JË‹—?¿üëåÇóÏi;8®Î[<óª0™MU®sºK&Ñ7éø½ûòíÛGôQÓg‹Èū½*ÂX0~Í®°¼“+¥C展ƒCªSK{£•M]`ð¹}¦k•b—K±\…s¸}épÍø&itoˆ\u]Í@©ûeÀuU*àmõ¼êºm©`z"DfïèÆ‹T;ýýpÒ¢‹AÏ$éÙ$ $¹EÓã{4=8ìS'©¶£YUá"u¶Ô­mÔ©—íõ¶;Á«Z1±4[üó%þªöîš,Ìnœlb”PÅ®èQzÆñ¹žFW-Pm;Ý# h†úÁ–ïÔLê‘x§jëó¼~E¼%4I—ÀØÄ| $øPŒˆïµ5ns˵¡•Y%ÎW"ºh[ºŒ}9¿Ø$“)N¦45fÁb’_— ïo4‡¿ºåjE ð­ÿåùìüüÙýû÷_jlíÊ®ìÊ®\W¦|1—îö ÒéÙpa ÎæH¿}Ÿ)ö)ÛÊu¸ZÉP‹Bè±® Î°óeøîQÀ,àvA˜‡Þ؈É^™1Ñ,R“ónAXÛÒðnrˆ¯ •¹i&“MŸ½{ؽóío6¿?›ÐÕ—t‡ªtœ´ FÂåd.Æ„ô:póëÁ n8_…XP¦…óEK<îríñ-»8€:F c ã&®¶‹ j©A‹««Ú@² LÙöÙuš&/TÖNžJðA ¤¾é& ]ЃV‹³§´<Fçã=šÝºG“ýC£½éŠ´¾èÖŽ@èK IDAT~y*“£0_˜*“7¼Bô1é=ö,LøŸ·7õM°fRZ\J¸a£¢‘g!kÎ>«Õs©ÀNÞôÖóÓ¿L4ÚæY{‚iÒ>î ë'툻Ø.Vq‹îM€½®a™h(ã Ãe’±ðl± çË%ùŒnM§t{ºGû“‰.€™ÂÃÐà‚1œKIcdíµ‡Mzõ¸¡‹?ù“?‰/³öïÊ®ìÊ®\WÚ¸ˆŸ7Òئû@áÀVá[CIñ×ÍÃvŒÂ/Á“âvx’ªk¦ødà†î5êTÃé‚1xRJzx7‚aìTìï럜§R§4Ÿ3"NîgD´@‡›m¿MŽ_C V ³ Á"…S.€ØÖ0\Ù×M³Õäû3œN Ü“òþª­TN¢Þ>]Óá^kÀIÑ”Ùyϼ]ÔHÉ×c8“#›Cß $bóÉ9Úh:>,%.†ýèà \8{]î÷Qþ¿mª™aª6Y4áúÌì(2˜ ¹tÊÑÃFY¸ÂPÖuÓ÷òé€m®ì‹`u±Úà8Û”òc!O ]i¹²2¢ Zü— Â´}bÁÜ` Wq` øÛ`;,98õö⡼§ù^¥]©ýŒ¡R%èá}hÈØ”h“"PT8þ:Hó{¤L|‰ÞŽDûppnÀˆÖ…ý;˜:Y×Å~˜­÷5<ª7Ñå÷»FC\¯Ù/%9ÏßɯÅ>!³émíÊõ6¦®l€çêŒ_¯ùŠò™%Dg±X\†n„ì £Š3_Êyüéò‡í.~ŒÀȶpˆm™IjGy“£þ¢Úµ.ÝòÁÉÉù»——«£ø ©:ÏQAHŸ[l¼yc®œÊõ° Š¤tømG]m`V"ŸØEôz ¯EØq@)70‰îˆëi7\÷ã6ím÷M«ÛÚÿ&é¯e½ê^¯$¥oµpjAtªkU" ¹./hyö¬?±WCJQÚª p ”(ÊéÜk^Y¯”šHûÜ{h?ÎEè¶S°Q€v%˜(ðƒŽZmÄÏ35ÚYw2»ûõÊ A»ÙèŒÌ‡F‘vÇ7¬½¡î{ÚÀÒÐƆVÐO^WIÈåjmW÷»ªèwÇöƒJ&[a_KV"ôh>§ž?¥ž>¦Gçg4_­½š¢uyòxXÉL´.]¿r`D”j§tþo~üÉ6^å®ìÊ®ìÊ«-“Ó{]\žüŃH ¯L‡úÐjShãf~ÀCXñ\3 ¨ˆ_Èø%׬°}¼x˜àvj‘=¸Ù«MìŠþÿ£Ht,ÄoIOÍïÓÛM\D*7Þá WéÙ±ÄÏ@Ô±ƒmVíõdÎ/‚$Σ)˜3Ù#ëTg:X˜ªÎcUÞ_4ÄÁµIt D 2`Å3 DýL³™¹°>\ìœL·‚´WAÓD¿«76J_oK©C¡TÓDjúÿ‹Û¦’Y ¦Û’íc¡@s±± |œáŽ=€5¥¦2¡)·GcöŸŽaÏÍN?2=`„Wm‹~J¯c‚À ‘d{-Š÷IÌíÑ©•¿ò¶È²63 @Gqf®D¸_aÃç~ 20O†0¢là 6!°¨"sñ³YÊÆ ½·X}{ŠÑ[IŠÙS`„ÝöŒyžY ãÌÐÉ IO0 ”™#¾æ 㦠Á÷7fÐMŠ6ß Ï3‹{_øán­¶~ˆàŸT‘V-é£.êµÇû¨ô¥b/Ó'çÿûŸ~üZmÌWŽ¤<ôóùü"Uþ&:ê”&1Ô¨xöe™5£äºŠMž«ŽæÏNñ³‡ôÅ—tv±¢.ë¥à¨ÐÉÃÆAÊéÄŒ¶áÅ´®CÐ;ν0b×ÿ2–øïmÂT«NfSÁ%;ÎØñàëCì§$öᶱí^$m.*×)/r¯º®/”HVdBãËú¿r ‘ü“TÀ”€£¾Æ2[(¤‰¤3*`$M_JŸ+Æîo`{uºã@¾åæbªµ¥.Ödc7hÉÚËNíÀð %ï6X¥°² €±SÖÀ²ycq™==uJQª¶Ÿ&Ò±Hw² 8©ª—â“gYNlíûz¢Fƒ¾ûY»¢OÏOéGÏžÐ''ÏéÉÅ]®VÔ¦ù:½úÔƒ#«!”@7‡!%^Ž¨=»¶ò»²+»²+/Yþ§?ÿóÕ““óǬ‚nWí-\ž°:ˆí¢ƒ´égò=¤îaŽ<ágÎLÁà `Äõì>|rÉ6Äl9ùd=Çb(´2ØèX²;n’3ÜÉ›‘Â[4d»áà5[Ë~âÀSN ¡CHß0€ ƒ/$¢íߨÆš€Øoj 4…¢ƒ¤¸X)„µøaƒØ¡•·v5æÔ­9Ä4Øî!&ŒêàŽ@»°¦I³ÔMŸÄuB`'†CÏLÈýW|(¤â"ò )˜Õä9ãi‡}Õv’:öjÃ)’ÍÇWÏì‡ê™°¼ûBCbtŒ{Ï;ûBk¦@c}¥s9×3d- ëB»»5˜ 00¸U† @œÙ>!ƒ<(“b•U[2:Ñû°2É‘)¼©™ ÀAâ¨ÿ6Øþ:ëËQç=… pi¯K›¥°_Í&ì'K†ƒóÓÒ3œ¶¶" ¾´Bú<ó?)':p[꺡dÔ¥z­6æ«dMÎå ½Àiü‹:¤5Ûà:Фfz\•)åºzlÒ´è$\ŠI}z¾¢‹Ëͦ#::ÓÞlD£QSŠ"b3äO\à¸\jÙ€‘œ©DV $]×tàÊÿCuÊù!H-Cü½‡p¤‰Ü4&ãmmE´Î ÙÔ'W}~Õ½¯Ó’¹ê³ú9_UœOŸ '-Ex”U,4ÞËþOäǾ£mf] 6Yú,¿oûRÈ+;Zc Í–¸W=Í|î!DH¹a,êdmå4;Uç–¼@FîÁ…‘‚é[ëê ƒÙO ±› L0~;“)ŠT‹¿÷C/é›]Jã‹WÙ|"ײ¹Ô€Hݧøy?¯EBÙ#3êü½Ö¿C9ÎÔß3]¸Ha[‹9=YÌiÚ4´?ÓádBûã1Mš†»!M¯±ät½€qj‡ ÂË “‹gWveWvå•f¾H·Pof¶—’:Óպȣï•@D±gTßE@Á)/ˆú[Õ"@ñUØFüYõNèšç¯ë­ë™*Ý¿Ÿ4ö NzÊz‘hú4câþ|É$—B¼b²ðz)(¡aÃs}?u;2¿°”1ŠÛ1kø¸Ü8„I @¸” lÅزì­F°ÜnE‰Àöl,`Jå†EM §bë7ÿ,ÈÂrd Un; “1P‚ÈúUÇG£]˜f±Ôúf JvLikm×’ùÑl‚ð*Äꇒiº§CFâ’Ù«v‘²RJ‡8·\n'ÓÄ#?$öJºëáKÚgfgä—‚aíOÂ_ TŒ/‡"ÿš=m‡Ô•oV­- ™…]¥àÖñ! 1YH“~'ÂØ´¼‰Ý¢7dø?¬'V^©Ò%Uý´:64ÀgS;0 ÊNÔ&8Hd¿Ÿ¿c~V€ @7©6’aB0*Ùæ$Žk«WU|'½š-ƒÈkµ1_)8ÂÌ—1Þ\ðñ*‰›\Óû¿ðq“b¬âeÓë™}]$:¿lébÞÒtÚÐþlÔƒ$ÓIC£ÑÀ{*6`ZG菱pUV‰„²f'^k×O(E(SXÍÚÉ6ábçtF±Iõ “ÄX Va‹='…}ƒM D @ÕZ#/"ˆºM´u ë°éº«´Qn^|Ư[€‚uܲucMÿï‹ßºã\É=ùâkXé3ËK¥[í9ˆã%·IÒ†pÓgUoø3‚VS.q¼ŒÛÄD[ÎIÈjE>ÛlÐ>Qã¡>$*ñjÖ_Ø&”Ÿ‹©|p*€Â<úÞ_Ë8éOCJ'!±“Z´ã dÛu‡(3Ý7­:×kfZŒæ¹?ïZšw=]ÌiÖŒè`<¢7¦3 ›’</J×SNîYñ„¨Ù1GveWvåk-M çý’#JkðC±è.+…+;®½\ï­d'°%CO¿OA¤îgEv4ðæ܇=´Ì\h  z‰¦[0§åS wl„dšâ“„è 鑤4ÀL´ìwõžº¯N¿äð“hiÎ~¡û2)DÅZÅÁ7*¡Iìê£Θî\‚æ¨bxwÑ)Cho²ƒ€6&XõïY†røàuOªƒíi„s‚9^ . SÑÐÖR8à:x2æ¤W÷"Û§ÕúcÒÃÍâ Z)ì«Ž¸ñ·´æ2@QÃVJ4}¨Î¿™ÀÐZ¨ÇXemq,RCI …4íX0™æ•6f]°}€‰Ö·YG‘XTkȳãƒc`cêØfë[Œ4Œ) Ž½”&ÝÖ½`Ô‚?§ý%…aBµÍè>ݾRŽ©ò@ÏçÂú¼©@NÂööŸ=$Kÿ ¦u‚`pmæ"(%‰v]wBÍ/1sDDž3s+Ò/Á}¹);äEOí7eyÝ¥áÒâ_ÍVÅù¼ëÿŒFKšMÚ›Žh6kh:QÓÀ¨¨Xq11Zhˆ ÂR-žT]«Ÿ$p$Uô²)EâZ‡ì«8ÖZ9°†ì»q¢ÇôàÐ&°ãºL6×¥ï­?»)(vS ’Ñ8Ù\Äq³i× \3ÃÍaWXðׇ‚¾§ýåàläºÚâÝkÆlG¶9æ4 •îQþò(Ò“)ÓS¡7˜dOyåVâ…×t}QV °Ø€ci¸Ÿ¬Hž=Ží?m»NÇí»‡Lïƒ-[=…ÙRâMÖŒºc \‚*n:ä3À¿¯ïB¥ñ­`g+“5êc_b\±t›ÕXoPŒ9cÚî¦úüòý* %ÿí†ûÜVñ3h‡Ë®¥E×Ò!jÆSê/=Õ{ûBžÓ,g»²+»²+_cáðYbª kXf'ÖKUÐ[ݹǕÒÖMÛ¯áÀ¢„4ðÈ”Šm¼Zý7²u78(åÆ¥{”ê‹•ïcŽ<C!DÖßk¨£;Í2’þ'}ƒåÀ$I`I\¦8µaô}ÅöèòÎîÐ3¡P(Ú™bú# E…½ å=½%¨©÷e蓼Úé4Nîœ<±t¥x/ÏxSXÍB ~^&10ûZ÷Tèܶ©6ˆ274tDàÝGY(¶³ý—`D:ƒuÈ|lh¡)YÞp`:&xXWÎ ‰¼ÛQA>Çg£AœûÏÈ´NÖþ0ú¼ŸWÈ´»§V¬ðp6ß C±­†*d&y1÷Ë1mm¥~Aò”n Ñ!>Ð&e8k{¸M&Àr’ ¢í€«“>ªÊ ë Waèè­âG%ìŽ ­µàw•OSƒ+eÁ5×G™VMO§{Ïé`öZmÌW Žt]w.")‰XÂ¥¿Тgý:Ÿ±íÿþo¯½Ø)ê(…#2tòªj[¡‹‹–FãЃ${#ÚÛKa7¡ˆç«Y= ’E•9¢÷.årÈPؾ$jdˆ46Õ`Ÿ<Œ(}uOÇÇPpùuÈ°ž.àž¯ 8l ºN˜u°²­|Õ0ß,éáñˆl×6tCƒû­ÀHJ£Fû¨O¥iÏʤd•‹Ø Äéx¤ç7my—-ÀH*1<%þä|Ä_4"{"÷o¿;éäA º’N—îC€u8õ´20ñdÎv?~YœvÝ>“ö£‡]÷óÓßbù¦y%³¶ÜŒìÍÇÿz?”ÿj»*›¶ßàF¶]:Т»MO–{Öo¾—¥ÒµWkz/5 µ­ôó@4 BSÂyéޚ¥Tr5$™Ö?'Xvýn~@7U‘vÕÑSšL–+¾+»²+»òŠŠ´«GÄÓç"| ßKÄ6œ’ÝÇì§è€^­ƒÅÞ¬«^[_LQ[s¥\z×ÜD^w}Y}F–cáœdÁFÏcðĉCµ/¦ï$“"%†Ÿ¦¶ºJV<+Hb¯ a•@iq4Q„^V,Ž ÐÔBT ’Ñ;Sµ§*èÁL¥CŒì Gª8ÃUB!Ä3ÕIÙiœ©9l ®@6c%° oÖ®cÈ @’ŠÕ­÷ÒS7c¶Ô`Ü^‡„1½lÿ×Áì=´Û´ÏµZÁl4.Æ/öTñ)+Y÷5µ”Ïצ÷1ãÏ?]›„#SCÏÁçawíSMÑ.èS’bƒ}µ-?ô¿š ñÝ-Bãuú„¾Šá0?´.æç¿Ç\e=-†\>ìÕê¸Q-0Ú}>•kƒ+¼^ኹb3"‹¯ê}-^Žó½ÅÚ•i5žÒÃÃ[íÏdz§ŸÒëµ1_)8rçÎyŒñ<†6ýþ*ÑUºaXÅ‹‚!/Â,Ù”ÁZÇDš¦£V:ˆ ­/½wÚw-—]¯O2›::ÓáÁ¤×&©¿Óu#DåPl@Çô"Ô„ŽšPÌ+¤H^ ËP ] ÖS ç/!@†üÞ^u›^—Vw[H̦”»u¿ÔŸ×?_Õÿ/" »­ppTÓðéhIëÐJ*¯9©x—t?¤|®‡¯”h;é7gæ¯òÆK‘´@-—!œž}:c9:ìèíŠß}#éaø©†«äWÑ I’.ΆØn‘-Ò®þæïÚî/W1Æ!ÜE&ºG›Q²¶ ÎRf"ÛÄÐx.Û‰Ae¿ª´š1Vó[Ft²ü}tñ=\î—c¡7ZY$®–µRõÖùµ¥¤k&Ìc¦8áÂu´­¢2üÆßKgàKVýDf¤ |¦i³ñ±u«/î,—/®µ+»²+»rÓ2nf'$ü„˜>ÎÀŽaªLÏ6 Õ. ìÏŒÞÅšf þÈÔýu¼ì3ðáÔFçÓn[ßXÓò`Š!C v]ŠaÕ~þÂ}zâ±PÜ# +"¹þ¨6‰_Š‘ÊÌ°º¼OÙŽX7Lá_:¼Z ûšj_H|ßÓ}lp.UxÔiÔ¾ 6>†ûG ¯°w ·,¾‰&†ègAÃl@“ÄÂiXL£„µIJPI;(°k¡D3]¸÷¤d¿ ƒÖA áówƒpOÒð©B ßúÄû0æðFà¥êP ±!}¯@¡ ù}íòµ¹¹¬Yý 0ÉõPÆ5Gô´Ý @ó)îãZø»Àû»žˆ£7ìéœÉǹ# *.,ÄG‘ø¾ìÇAÈ[‚¸vOŒ µ<…or(Hl T Œà\¯ï@Š†{‘2* >>TÙ˜‚W¸ïêÏ‚ÞE„ ~ÇYørºG?;ºM§³åe»úbyçõÚ˜¯™N§‹åry’tGZaè°©\%¶z•ÖĦò"â­Z¶6›Ø éß–$1Gú+Âbì´}£¤¬Ô{qÙÑå¼¥ç§K::œÐáþ˜Æã™"’õ |±14¸ØèÊUËàHv4 ± –”-Ô¯2*Ì¡uÇM/·b U¿Ì5unš&yËd›@jý»m:&›~Wÿ~[:á)kͯƋíq%A8F`i(¡¦S›/æõ £4P W¹iúÏbwGüºÒ‰¤´7óeŒ‹³Qódù‡DwŽ„Þ ½7Žr/‰§n²œÜ+Á.ÆöRâÏ~Ú¶ÿégÒ}±È!@>©ë5S˜GgZÆœ Z† Õú`Yû+ÅvEµ) 'Ü€mã>}9—~pñ´“B]_çhà¤Æ*«ÍfdÕ—úE`ü4"ciÔ¸0´‰Öß‘h}?ôŸk`f89Cé¨æ!Äx“ ¼8zòàèèê—Û•]Ù•]ùŠ%¶‹3¢ñÃ|è_`”ÓlÀ@¹Ô•®îÿG†×>æ¸ O•aßfØÛ©Ë,uüžÌ®C-©ìÜÓ­ò„ Úø~ºžCÃ@Š×tÏ0¢“L"ñ!/ˆz)ÄˬŠYx`x<¶êTl›â¬hcªƒL,0 Îä)ö2b϶#T °Ú[óà¹KöE€ »Ð¦‘’ë¡~a„Œ1˜ªZ(’÷ÏžM®L¾Gq‰9{NÌ'åÌh Ï/Rç¶-ÓþöìmÏ´£´ Üký¯c)§îò~oÀˆxÆAÝF2ŸFm ßV°iH=ëš*d½åF †Wi?80é·jk9(R€ÀHÑzÁ3¬ƒà`RE’5ë-³ƒ8EŸæy)ävŸ->Tê½Ý¶íA­$0q7…ÑÄ>{”äq‡$lDóÑüwbŒ¥â°þ2V²>ú³¦´O†b2 ¥Áò”û \=I³P)SÅFÁÆÿÃúëc¹Lé}Äßz]=z~p‹~tx›.š†bà…yrôàõÚ˜¯ù­ßú­öÏþìÏÎ¥ë:Þæˆn+×òß䳫Â*¶.õµ5«¡þLŽMß­ËÒÙ-‘0)*8sÉMDóE¤ùò’žŸ.èþM&jã‹ú0 ”2•'jŸ".ú3&w|£œŽI¨X“PÁ06YÅM G»m°qôbPÍhS[¾ˆÀîU)zoʹî>›êH0.^ Œ¹7Þëžoø¥öG݆¶¸ÕXz_«÷(©Âõ‰P¥¬Où«Ûß>Žôí‰È; óV\é«N†:‘ù©týݪý‹/bûlÓó~~9ú›EŒÏ§ò«“FÞå@3cN‘ƒ$>:+a0˜£ì¡›… éÑ€F£Tc¹ÕÜ[tGô³‹÷è£Ë7è26y›Ãû:‰‘SN\ñ¦«€«€«Fx’&\MÃÅñ†e™R²™êÏ'M°“']KzJoaV0΄hþWŸþùÿÇ¿cŽìÊ®ìÊ×ZžÎ>\ìµ1ïmH™ÀõÑs©ê0Æz/­¿`?× wålç¿=tÇï­6R }ª=†ïÚ  °ƒÃ®: 1Ø﨧²ê¦ìc9›9uà0JäÂ>T‘ÊÐ0IqM!$I’ˆç’ j‡vÅi=ì½À¨TÆ‹À÷µ!K]æ‡÷éßÁq3§WlÙ1u–‰^ë=%Æî !KLð°?­w s‰ª¢Î^mK+è£0TF´+›üû>‹dä³QbaÙ¤lmîBЕÄP-¿+Ó¤2 ‡ú+ØAÖ^éú,Í õ;ú4¿`G®Í)ì*ëDÍjlaöÆåÜ .†öamV;TÐÈzTA% XGc,ÚÂì쪿{oð€E¡vZî6­ Ö§ãâpDD÷#ÑaÌY^|¬ü_ÝÜ”ŒR•Ël‹ÒLô60ÞìÐ+ÙoÒEÃÙ¿“v•€OÉÄ3´ÊŸ @G Žà÷ §²¿ð½½—£ }qt‡>™í'­‘ÜÏ4ÿÑÿù£ÏÿôøÓ×jc¾RÁŽï}ï{1„pÊŸG†›W•m Ç¶ì(7Ýt-†zls¾ûïHˆÂ‰€è—²FÜÑÍY-¨DuÑ…h±ˆÔvåÀ/ãá|£ì‘L‡Ø0»g ž˜YNÆi§ã"ÇmC*P* ­Æ/f24{›.Ìuésë>ùÿÙ{“K’$MLDí½ç{xDnµtVU÷tu÷`šs!8w¢àÞx#ðZ¿sj€ÿ §@€¼ 0誙éšîZs©È5ößÞb*„š©ˆ|ªÏì¹GfxDfõÓD¤»¿g¦¦¦«È'ŸˆŒÍ“¯w¦ßFöÞ=<ý}²Ø‡±èÝtØ(µpF¥˜½ÒGÂåwK꿳[Í®(ý±øŽàp4™¶ ­×÷º b‚=ƒ0ãÛ‹½W>—$ô|t°¶e[¶e[^báÀ¿ï‹³Ÿo2±-+·M‚˜ºJuf22ªïa7`í/À‚ó›Àz¨¹Cj0)âhU¶øŒ d2`f–fˆÂÖfØôÕr¸”1#¤ÝÈÂ¥†(î…ý+±ÜO‰ø‚{sÓ16¬Õ”Öþ*ƒË˜ý.äZ0¾g‘*ßΣžO!抂22 ‘Ÿk,›^yŽjÈnF’çƒd€©QåÜÜr;s\ÊJ½ÙBÍFì~1鳦òºïÜ\ÈÛëïüˆ;¦‡}'d®+DØ×Àæ6«·3x,>K•7(sBú™¢lM6£þm±Q@ùfwt•ñ5T²¤ñÝ}øØÆXg5ø ÷Y<4D*Ç*Ô5J »ÌÒ}Þèº=8¦`+Ê×_-xµ&·_ }æ™äFóN¤x»g‹ ÜØ÷¹ƒ±î"³ÎyþøÇÒ1¼ˆÉ\Ñôe”™ãñsÈØkØVŸŠ˜Þ˜*¶ƒhZcwßÃ*jLØÁäRç,uU^«Ã_–é|÷€î½Ag3è†ý Ëä•Ë˜/aæK¾FÌǹ®KMÍö¸Ê]g,^ÈÐOB6ŲH;EÂOBQG*0ÞqpÛü`ïB‡þÄJÌ=`UC)K€Dë!;T#ͱŽÁ§Í÷Z[:nEÁC|½jÄ(ûûºÃ$T )û;ã*b¨ÿ_U)Ƽ <}gw÷Ö_¼ùOnýàÎ9NoÇëÝ(¤eD|ßjÈúÉØ#(Œ 6ê6S"n$¦Bqá[û6(Ê›õ¯âö12†‰ûì´ ¿Øgþía”wn‘üèiÛÆû1þòã¸|<ö¼¡¢×uõ.èÙé¢ùÅ~C¿=œ…wöväOö¦ÍŸ5ÍjWW^mI,„Lx×~‚x>¹W±¡‡—?¤»óïÓƒå¬:ÜóšˆÐw*œ§”F²\rv b„ŒõyÝß ñ.÷¶ ¸ö•ê¾’æ8ŒVÏ›6K²¶ÒF®U.Í% :–m˶l˶¼ô"Â'Sì)ÚÝ­dêNe0s ,7NýÊb_ øÁ,Ô»½ÿ^Ÿ#¦–C&13~ÀíF,´¤S!CÈZ¾oW2¦YÖUÆôßñn”EzY“Š€®d`‰žyE×u–íBñß!â]"IéϹs·á¹xÌ•] £ìa-,‹¸ªÜ+Äèɉ›+Ã*´±;«Øœ*øRôÀÿÎì â *ó*"w€08:Xá0 È|*«[ V×=GL q­ÖêŒøœH,ƒ˜eÈ@Αìzå¬'ëm,‚,ã(à Òá>¾\žhÜH1‹g•ÕaëNØB1#Ú¼0ºø@µby‰ÁhˆñIĺªóêe[Û¤½3ò•Øx’kFû–}¿pÍͳbl–îmSû‘ä;B”‚¯†~”ÌŽr¾›ñŽaß°¹ã×0㛃ë÷î0 pD‹OéëšÁ%:˜œ|œ#)sS "4Rƒ$®—ÖQp|ÒÕ"}ù7§x"ôôà˜îÓI˜ØþèjQ·>çÜ4ß~æHŒñ$#¨d]'hê ø×}>ä²Q+Ôøs¨ xï&ºîäDÐR¸èÝ4ÖW?!-à#“¥ú%@ÞjíªÏNS¶'JqoÄÁUN,j”?×¾„(Ãf… >…sP,Èe2™]7î¦.C÷½¨ÛË× °J¥òÝ­ûf6iv¾³ëñÎ?Û¿}øÔ„}» Èò¿nL СŠRô1v=&†4Ãßú+ù¸šxTÔ¯G}©~ÎÀ»_Õ7›Ê&å\äòœéî¡O§qÉóÛ—RoK—gçt7\N¿˜îÿðxo*?=ƒ­n×&˜Ñ¥d†Ê˜¢_ØÙ]ùÿ›ŒI ˆê"2%GdÿAD4ðšxh¯0%¢ãH|Ä]ÐV9# —DÊu9h]FÚ/ªL(¢=LP6ˆ*v,JDbN³“;¤ VE—É™¼®·} ,› {ÐT Ü šxßVVІ-¶‹ÕÂsÖ?3ÈÓbà•éQe Ì¿¨gTò*‹S%OYl¸EŸÓih 0Vç} 5ÜJ{§ôÊeÌ—Žœö™hwÓuCÙdÆØ CŦ¸"C÷Ž•!ezS&œªîbñw‘ÂR¡¾ƒ£rí0o#Ðá¨tjÒ°Ò£ ¹)Çy1L‚ߧÅÁ)>¯ëCô— ;h@  Ó¡þb}\Ç¥©þ|k¤þîe²Kš½I3{kvxøgo¾{ø½[ÿt¶³ó61M ™®â† ×ʆF­äò; eà "»wÅèT^ÄÙlÈŒé7©äÜÑ’´NszÁrÖÈ&E¿¸Ž9ÎóX]çú±¾ÃÒ[ÄÇpºœÒãÕ!­Þ¤ãæ94Ïh.©á6‹R~((‚8d·íS¡ÅjJOè£ÇGt÷tF»·š­­ª’ ². w'Ó´é½ëù¡Ú$оÅ0²ŠJɺÄפ:̆01AªáÐê&ȹõÄêÒ b”åì"ÆËk ܶl˶lË×,ÓÕ£(“¹³*M¶&`ì±êëTEÃ…_ãŠlÍø3‘ñq%W¿[g‰Úy’ i®Pãf> cæ/b%c’èšR`>}Ÿ˜r)éŠGÂþOöIL1}’äÍ¢,è—ø\HX²bÖíz6yÒÜ8LŽå5 Þbe 2gPŠ2?ûß#dˆQå×#NîìëÔƒÜi <á.ºéÏ¥tëòv€þi쬗@éÕçÔ`Q¡*C& ™…»Š±7¼QÛ1O´/²xŸy`V½.ŠYìAÆ·ë â, êRÍáÊHÓÇÝ\ù盳¤tQ6™Éç²/ÒwBf7•Y@&U¤°>‚6Îñøu¾<;LbNñ‘èŽÏÄæ’ƒØ3D`ü¥{ Ž·Î|rW&‘JÐÅ×'¹çŒ{¤¿•»öÔî1¸*¢㨕20Þ«Þ ,¦3zxp‹>Û?¤ó 1ûòõ¡'öþoga¹óÊeÌ—ŽÌçóË…þ=–Rw€1ƺw#äºAY¯jÏØgÉ ¼Š|#‹Mœ¼¼u£`„õàÅÃ6{óy$¬´®&¦ÖG|Ù§‡*º×ø&îô7ØïÄ·©âðuD„'Ì!´íj ™ëךͳ àØ”rwSf›¯\šÀ³·÷öŽþäö÷~püãÙÑî»Mö ÁöñÏ›Y(@¿"G¡­šÀ„ p †ÀFœ?Âñ釲Œ&Î6cÈž)~µ‹™Ÿ|"‹'gØþ¯ HÔeˆÕ0V×€yUÝWµ#nV$SΑٟ®öè¤Ý¥¾C‡Í6'´×\ДçPÒyŠ¯6=dE:›Ïèî“[ôá“}:]…~܆€ŒâwH¢€ãÛ¶‹µ«wNtnæÐÄvÏ-š¼Y Ãtïr˜Ól5 kõ^Rý#˜ƒÌr‚ûý¶l˶lËM–0Ÿ=¡=:ӽє%<ëƒÓÒ=0oŽ@//¿«~f?Ã×ø榀 |nU«Wµs ¸ àÖçrg†Ø³¹Q»â£÷¹õÙ"ZàÞF³ +)F;Ò/t凜–¿;PšKŸ8©6K¦d›Àqb̪R1|˜5®’¯0JV,ª(ãthÿDB©VÝ‘,O)ËšõFyÎUŒÅÖYG2‰ÒëôÂuňx¶N4èØZ3™fXÑ™£¤†B&FÄÚ£š”º ûs¤pÇ(XÄ6ÿ@G³‡2ÉDHö‰ø­HñPz·r Y®â{rG¼õÇáŠ(0ÝÂæÚý Ø*A|œbEøžåL7 jÅx§>wm{ÃÞ°¶âD©eþ’™V̼¦¡³Ù.}yx‹îíìÓŠ¸šýdT°4ÏÓ“[ÇôÊeÌ—ŽÌf³ËÂyŒñÍú»¯’Ud\¹ øs·*›Å^ \¡0ïv°ŽmVR mQÅú "ÛÒ}ù²¹ÕØ$Ó4Cžq¡y…(t“sÚļ™€eÏÀ}çí··àȶl˶¼’²7¥ç-Óùu¯°ýw8V:½i-úEq&›EÖm†r’+•Šjáƒ(tñƒðòúw2ñLÏsTQ@UY¬; …«T¼Ê¦qÙÊž€×0…7+Ï JYÛT6r‹y¾&…ß‘ž=’€’K¦ÎËuÑ£"NNSrýbŒ 8gQ– …dëŸsÕ™fÏGªj|´©"ÃܘÈ ¿ÝýU˜Ël”:ʲ!–)¦Sô$C<Äb‹PfŒØèŠÆú ¨iS=7!z+«Ñwãà3ó;:ë'“ÃFØàئ Yr F­š€àJQ‹0¥ÁÙ»ÑÁ—òlsÞëÖÕÌÕúx[w¤b¹`£0Óí~fŠÜŽK™Ö]€'ð\oC@qÖ3€SɸÊôeè(T »úÀ¸RŸ{y„sÇùp ØQŽYÉHñR·½Ü—¬§«kúŸ«É”žîÑçGtÒLñTªTìuå9ݮڧN ùùU”—Ž<þ|qûöí.²ìuÕëc½*uëuíMeSpÖ¡g´Âó„‡sïyÙ•Bñ)¿>8RïíÓ˜#ëO\×ö³@m³(ÇYãiE׋¢>FÝu±òGµÍW÷wo?gÊa › M¸ËP^ï Ó»Ò[‚œB²~P†lUŒ9ú¾ ÀSIË-ÉÅÓåê—œ>ÿÛÿãÿþéò'/q ·e[¶e[ÆÊÙ“£óÝ·Ï¿4ðRuc¨A_Y0³CÁ˜pßøµ½˜]Á-\#l? ÚjÖr@gAýÑݧ«³ÅdÄòP30¡gLÁÉÿGSŸ†â‡gªiÇ Ã¨vE¿·&qX¿iúܪ¦B4å>&C²C̉ä¢ÿ™¤çÞˆ¬'u7áøKÇÜV³Ž×GÖ emäþóC¬ƒ0ª£hµ*”}í rÙ÷‘ÝÐÀìî:¥|C¨FäàèšV_lµÉ§«¶Çc‡øsjC®°3G Îi~=cU8ð#,ksç*“¿/$F)X;²™zBn”…åälcúÚÃú¾×xàŽ„ãŠëMÁ¦¢%X†V´Rð/ïÑÜý9§>5ug:oú¨yö`µüÅ?¬–¿xxy~—1ɶl˶lË –ÿýßþÛÅ¿þÿÕ—Ý©ìÚ’Ç¢*&Z)ÅW®È¸¿ªb9s„Ü%¹ê³òõ20N¸®@ßÉ ÛFá&‹ç•1«·t=¶ûÃRg¡Fª•j¬fu㨂•PyÆDK\÷fÕ„äŽLNS¢æ@(&¤‹O’ÜnÊ7u…¸Ìœƒ6jIòã<ýo‰1Ȳ檒çB`¯ çž—ûî«i… –@™¡¦pÆt½:?­õ ›£ZMËtSÇÈ0f|€²ª ÃÄ*ô`©ÌšÀ%QEÕ3Ó88Ó7ƒµ:¨PÎëb= @I&8Ps´œ7‚õÄÁ9.I€¸LUÿåÿipf¯ÂÇ5<ȯϹH|@Äo ÉQ$™Ô@¤²‰|mkõδðyå2£ùªù܃þ3€*€¦†±a u/‘?‹í}]ö«]õÅ×·¹2­/‚‚~/—[ùó´—ljž¥ «ûGt:ÝIžu….£˜uŸ•>h/—‹û?ùÉO^¹Åü¥ƒ#Ô+°÷¯ø¾ø»V¨¯ŠrU€Î±2Vï&€dS†›ô]§¬ôØw÷“ÄËP`¦9%¯ÁlH¦²,ÓfÕe]Æ!å(Ã×jP°Å.Ó§!›¡ð%#nJ\ÿ]_sS®2×-1ý·$ŠóH«Å’.NÎhº3¥Ý£=Ú=ܧélæ..ùðIyÿgûD“]¡Ã·„æ?dº|Æôì3¢ó{Ú9‚"zŒ}n kJ ˆI±UB©Òýõs)®äòù¯VÏ>ú{:ûôQŠŠó"}p]`?Çû6¹ÉŒ¹Ó\ÈlºïÛ&’À:XÎ!8(|³O+ –ÜÐeDË Î÷”®M37͵UÌ1ê"]J»X ;cï²þ^)["Ÿßkøýc¢?Úiã§$o’Ð Ì6tZÓ­ù`þÊ/ök{ãw—ËŸ}—_Ìc|¾y::pÛ²-Û²-/¹$0öýþ›ß6“,›©°™¹4¡ؿ`C,,§ÈЬ ¨¬äA¡ñëê“ÙîCkªèy‚L­òÿ-–„þ&olÅU2&(Vd7A|<<£ºDee€²ˆÆ;WÂ*ëýšù¹”_C“åšdÆØ‹DGDt(åüç[Î` ®ZŒœ[]8¿àŠv$ Д<ÈSZ,*@.zþYZ`"‰ä,Q{S.Ÿ_èá˜V²DÖ¤yøçKÈn>B uïÞQå1_Öë2¦Î…VȳçHÙ7ë¦Í.L¬™€TþåröB„`‹Ê\ùw ~[*@)Þ¹H5“u¦@Q¢Áóšlꈃ/µ²O0Ÿ˜,ÞŠ‚ùM·‰ø;9 ºèäÔ×lé;m½²S°+|ë1·y“Q3bs0Õ aÃPŒ¹a`‰g€ùû¨œç¬ŸŒXàYq9ç0άW¼‚BVNu/'SzxxLŸîÒE¶WÚ¾¸AfÕȽ=§Eû½†r#àHÛ¶g!„ȉã>ÂyÑ@5`1Æ©c]ŒªûEÜ:ôÚUË+^ú½ê1‡þþ•ïNNR%8QØb¤&µ§8¬GHLK7P&MŽöl›& † ¨Ad½bØ$òX )­" !L„ûÔ=uÙÄƳ×^¤³Ã¤Ì.œ²ÌÆUKóó9=xBûÇûtpûˆf»;¢Œ'Z·M˜ö‰ö…n}7Ñ…žÞ%:ý”).ù-úæD-ŒÙ˜³Ï™þ‡k¢´²Z<]?ø¹œüî÷[d°qöÆ&cˆsÀr ¥.›®Áz#53jcx³«2HÀpæI `ÔI±Îõæ¶]vªjï¦ß±Oá÷øD䋧Ì_N&Íß݉òý[1þx&òn ÚWs Û~AÅO²s ³h¦¡ XÈbžØÑ…²öTÚ_ýÝâòß?mÛó61b˜—ÌüüÊÁÛ–mÙ–my‰E"?L¨sÊÇ•5¾ß×0SƒK7nÐq^¸Ôgœ^³®Qø9=äAK~š!¤ØwK¥˜*¹ËãUq6Z¹ÒPª®”1A±´½>ª"†Ï'¸¾/ÁšŸk´X  ºufE‹\!&…aŠ:ýÞî×»a*…^N…ø”IæRqN¹ .êèCâÌ!·‡>ƒ+²­ë¬UõÔà)t $p ²çxºšBFVÅÛ%=€_4ŽIF8Xƒ¬p¶^³—M£›¿k4áA&‹×Ù»Ã4p–Û‹[ÄþOH]‹îZVÒ§3Öæ¶lô|‚ˆÄ¼I;ň*×çgÐÓ9ÚØ3 GËýéZ·® Wa8a.F+[ç‹q¬'‘èM"y3Rœöë±±õ£ ec‹õ€¤1V2¬ºÄxz•]ÄÖk¤ä:4(­ÀÈÖqQÃV\Hü'î™4ø{idÔ –•~·˜îЇ·é³½Z­¼y GÜ[u^«7Ò]´-m$[ÜT¹p$Æø4„Ðj ›É(2Ÿbëc,»ÌUÁU‡ W¹ä¬)ò¡C.3!P‰8ä›°­«j5h@VªƒuÖîðÝP°D{n“ä@‹Â£9xb›$Áf†’î•4(Eût󉛺Oë>[k[D½H|š›*:«yë”ËÜÕQ¨->|NgOÏ)¹ÛÜ>¤Ùþ.5MŸ&¿éà6ÑâÇLO?ôìc¡ÕÌ j º]©(»+Î@é¬e±m/ŸÿzqúÉß…‹¢Ò® Á‹²5†X×a…àýc@Êuë~ÝÀ÷Ës–!J —¼â1{ˆh#:žoîÿß¡ßõwTDj²ñŠ) UœZµ‚×WŒü½v¯)úˆ;¿YˆÌï1}ô° w#ß9–öˆ~<ãð†ˆm7 íÅø@õùëIóß@”6ÆóGmûó¿_.~y"q•^ŠÈ6ï¶l˶¼ÒI>’”ÿ¤”aÁøã`7™RÜÝ»A˜×Í”zòzJ«?ÈÁg[Ê^p:»?&O©¡Ë²](]䮯#c`TzÃåëê·Tƒ0p¶ù}¸Ð³ËÎ1•c¥ªßË®µ G¢Cî˜$áTH.ûdêb2©¶=g•1õAë7 .žÏýˆg«x#|X%Êb!‹÷¦™¹#å Orì>ûƒg·´`¬LÖ¶œ!¢»&’|E¶q„˜8“˜šÎט?ê’çczÏ õAC5[ ,„õ9C£?®T–R$}Žk=Ä<ýsöt&ü¬ôéqYúž| (ú.mGy)ƒDˆ™ø!º%$AŠ LölÑì3±O]ƒ'Ð! îàÊ€éN ø]÷qP·²õ¯sƒ 9E÷Ü4LZ]c…ô²b^“Îן~߯—ZÍë¥`–øÏr«sY7R ó=úäÖz4Ý¡Öê¬~a'àusùgÈ0ÈóÐ.ÿpÀfž÷q¨i pÆ®nŠ/r B´ÎöªwX¹Ž›Dj)Ê2­> båy¿úàM•b;„ø÷•uVÚ.WŸÝb— S@ ^Ñ”¥˜ÀzðŠ¡¸~Á\00¤R–k¢Û¨S*ßisE¯]^Ÿ‹Mr?ŠžØk,Xî““äòùyçf³skŽº¸$½ÔPÆ‹I?wöˆÞùc¢Ûß‹tú¨¥§¿oèòAcªsíFáˆkɎȧzlçg¯ôÚ§ÿe “IÔ6°@6±L†kàÍH}uÝCm)A ÿl€@=¸7 9ƒ|)sUí¾AF±n·_èbŽ  PÇ"[{VG2Y¶qIÒ®½³>£#{—Ðd|Jñc/B8=gùQóáÛDoï­äöIþ¸z;PG?çÂJˆ}ú7MëÎ-Nì”'Ÿ-W?ÿåjþÁ‚¨F¨ßW{{{Û4¾Û²-ÛòJKàé©02Ñ}®)FÕV^(Ê\(¯õþOT²Hª£µ¨«?ܹ<Ç©TPª3mí 2C”C_UÆ´“Œ{ÆÁïθp´¦`wTQ-x¢HÕ…È01DŠ1BšÄdT1 8ZÞlé0‹=¡˜Rñi :Mq½ïŘe†{ïƒhà’¿³rA{ÜI¢¨îEùó <É¿ÊŽRd‰‰9й›Q½)ŒÑ»Æöír!Qšû\þš1ÆÓ÷kÌ/¢n5m6( ç²ƒg$êAæ ºktXóX´œ‹Ì:‰Z¿¸¼Ì`ŒÁÌP8¯ûw]i{®º>‰1Ú¿Êú± Ãˆà˜æþñËT6s€È%óDµ¹E$ךּÔls×g™Î&Ž¦ÛÕuæ I¸ªé›-–O±¸<–Šx,$Và†ëuîíÇ5î{Š2GÖfx^ƒT|î2&YXcê𻢷IB ³ÝýyØ̆Zåêgtãºá€cŠƹŠHsÚ½sA¯¡Ü8’¬ŠNëŽ/‚eSÔM “!€dèïë¸z\˜u(nŠt0›‹Ç’>I>S}ÓÁC $ð+Ãf¡EÚ\/*ôkˆÓ¦qp¤Õ lùó@0@÷Œ’ ^ÓÅ– åÓ˱{‘òºX%ÜJLYwDT&`ظ|³)QW¡ÅÅœ— :{tB»·öiÿøR–›É¤É›\}z¯¶MódEû·…vVty:¡ç_6tþE öRvk°#lCi—§ñùÃX>úÕ¯ââËÓzŸ`¡Á^ErHRÿ=xÔì‡1@eÃd¨Ý¾¾&³XQ…Ç-‚lùú1cOóû‡@ )ë»Vþ¨Kf•bŽÄ±ñë¿1† ¾Ó+'>¹G¾·KüË[1¾}Hôã]¡4Ì"â/£ó3ËMdžãxÙ¶?h—ÿþÃÕâî²d~Z !\üùŸÿyûþûïŽÓ¶l˶lËM”¦Ù?ešŸ”â–[SiàsW˜©ÜóíP‡ß1» nó‚À÷ýÁ-k×®Aê2P|uy ,ð•Öîç@ñrW˘L™‰‚q ¸²Üêã\qJÀEÇ0©É6êª ¸5‚!Ù¶(Hbìï7D¸KgªS,]»D´;„“ä<¤¼|=+À@ ¶çõ}êc‰| Ð l—¶DÒ:¡Ÿ¥:s­ÿ ¥]cpôÀB@W4ñk`Yk˜‹ž©îI#„H Š¬þ˜•;¡˜ðªÅd@ŒÕ%'šÐ›³‘Z"Ì|F@$¿3b öDP´µo¬ËóǬiˆJW .ŽN‘˜@R}K_Áõ^aµ„ìãйÑDâÖ…*ô¦ÅÃýŒà Êt3S1{Îwe7Ù ftÙb[CªÚZ×ä¤Èrr ²Ô)õN1÷=eQ>׉W¬uÆŽÏ«U3¥“½úäð˜ž&¤mô!°FƒJeß×û!wñnâš=y-¸Gvvvæ‹Åâôºn2›\[êk¯ºŸò!}Èýf¨xí@¡‰-ÓEŒ"l§ß´‡Ã`•Èa—q!UÒT]Öl¨¨~¯…3 ­ÁM ƒw1­ åa¿ÎñÑ «h")úÆ5ã|£ ûáç‹ÕW6úÉZçåM;agOéòÙ9Ívhÿøˆvv©™4pµXõý¡ aC´w¼ Ý#¦Å»ÎNèüÞ„–ܧ¸‹¾?ÇÎi2¶ñòò~ûì“ÿÜ>ýõDz|¾¶il3†€‹±ïñ³«ØX禺Ƙ,›Ø+cŸ2Ba*jîð£ÔÄEZ<Œ7øq§ïBƒ’¾™^‰/ë;¼ÈÂ|­CÇgŒ3ÔŸ5X2g¾¸ÂÝÇÌŸïÇx|éÝæΈÞ`‘ݸ‘(}ž’îâ¤m?ýhµøùÝÕêþ0BýÚ>ýÍo~3üÛ²-Û²-7TNöÞº<¸ÇLÉ°Ï×Àµ15Ø‹’ n¬þSÃüÆ5ë-ÒPq¡õ½¾x4ÊU–Ö^¡+1/(cB,,®®CÇô¶CG±ºĬœ&‘X?’7- RH-ïk2¾ƒ'6ž €vñ vRZÕ–ø"$L”2Ü´¢²RVιdžºËÍ탳’o‰u5²GÄÒ+J ª·äTÂÆÎÑàè =Ëeÿ+0¢9#(•ñÝLÔÆqpR¹•<«dÖ‡ÖïËO'V¹ÞÑ{ÉÏmºÀ¬”ëó ã,#L\{k˜Ý'Y<ÂØ(¢ý ®2ÙƒÈÈçhe`•Éoîê¢AFºõœ@‘[‘ø-!ÞMn4TÈŒ RɾÏèº/äI½—ÅÞ¥Äÿý÷_ËÁµ-Û²-ÿxË“'?›ü«/Ѿ³vž8QXºkD 0ø@]È€ â¦…†R°P±k(0Q휋2¼(Ý fÔF1|Õ!6‹Ò¬dÌúú²ß v*“xߺ²Ø»ªªËú‘AÌZMøÌ!9@õ©XžßÊšîÄîßÁ––—LógŠ«’åãß_|ðÛŸÅù£Ñì ›æû&wšM×lÚ×Uêkqs¼àò"Ï+¯mR§™£æCHÙ¡àëqU¯6êt¸… 6Ñ…ÃÜR æÀ{Òñ]lz_쳫޿3±6¹èXÂœ2$^.Bøì™È»Bû»$opxã²]^<ŠíÝG"çš«lÛöÙè"Ù–mÙ–m¹¡òÞ{ÔþÏÿÓä³éJ¨iãÈ6{·JgvWq!¬«â©®5ë)ô D6¥ Á SÊó‡ð¤ÑÇRÚ:vp-Sà¼CÓcb@°Ñúªýú³K ídkçUÙJeÕÐýfîÏ&”™R¾Ÿ¿‹Q©º$öÑžPh…brO¾ì£Ê’ Ë °-@‘51 :¢Pñ*pÊå|¸Æ¾ß‹ó7¿²ô,cq°P9׊ÉÝ¢¸ü\`ÊéhE¨Ó–€O¹ü¬|½.£+áÛr^ÏYh»C#ÀñP¶‚'†¥”ÀŠÉ_*›‚;¦ýuðhUâ,}¨ ¨”ßA¯îíRHŒl5¹ÐY: ™ŽÅ¶ µ¥1(0Ìì5ñÎç¶MýjX­O¬dÎÕ Ü@!ȱ¾û•ªˆË²"^W‰á²Ýça¼.ߺRšÞ=Ü?¢{{tÚL­µüYº1+)¸¾ª³ÌÔ"ì»$D/ã?ùÉO^‹âxSàµmû$…¡Hþí›Ø×eˆ¼H¹Ê çªÏǶÆIï*ai€1LW¤{âìEÔ_˜NØ6Fóos 3ÜW-ªRé”*ÒuòS„ S%SÏÐaŸ%Pèu´ßéQ’ÖXl¢o+ –4aón¹ž†[7'¼ž5‡Ã¶þÈ7G˜^ñh– ¦ƒH³ýH±}B—NžmF¾NŸß$hqÕg›ê{ÞXÝÒKT;ŽúãàÁßLîHiëG¿RŸUt²p‚“að4Ôu‰SBŒ—s}ºýÝzਠ/ÂÀ©ë¨K$ŠçL§çħ'ŸQ»Š« n4UIŒ‘“k^»-Û²-ÛòR˽½ýO‘öhµjöçKš´­1ÌjéeþQº·˜a”2½4‚uõ¹÷µ,eÖþZð«ñ,TdE5W@©ÂBA.z^TÆÄ¿¹P{Ëû¨°€ëµ€„e ÜlT¸¨¨FÉàÆ‚/Å]pðýÄb)]ŽÔ (¹ö$rgb“ÉBˆP’ þ÷YRŠwɬ ÄÌŠu.…ýè*·UÍ"ró.¨<sRçWlÿ|ò”º.BÊn¨DÍüäЙfLVPsÄ©à@ jN±‰°‚¹š± 槺3Õ³ÀK„Œ¬*› Æå@•ÖÁ„àû|ÓøŒú:ÈR¢¢ðv$¹Ý³E”ù£÷›n¥sU 1¤ǧœûLtß[®a|×å+$ìª>hÝÛª¼g“H茟Ÿ>Û!µ2,Y­îtؿEOwwé’¼O1Œ… (Ü F•Ô/u^D’/ÇßäfË#)oŒ±1NêØ uœë¤ò¥¶Hý=u{>–¡ûëg4M U¤EšR #«›’Ÿ±ˆðŸbyš°¨í¢;VéÉô›BAB†@bŽDP°ðú¼™ê¸˜¸ Ï¢=P×ú³iFîÛT"µ±H*¹†KéÄ-Eµä× ÀÔ¢~°à¸–?lÉõ7“”C§½RiÕy|ã:,†!æG}ïu\`ÆØ î]羫Úߊ„'3§*°·&9úwe|ã\Kyð:’ïõ5OÚõôaFƒ$û}f uæºëÛÞ_”Ù7ŠZ&žôùaìSôJºê3+w3"1ÝM*ûwó‹Ê ù8¨Ö6Aû±€±¸M-뱬ªVDÇ›Š=¢êÊ Øí‹DƒÜè×ßÏVe]ö`·©ÄÐÐÓýCúôà=ŸLSl:wY‹åþi@tÑt÷@°ñFý²€{–Ÿœ¯ñzMå&™#©Oÿ8‹×1”fSÜ-5@¡¿eµ¡ú÷±Ø"õsê6äí»%ÑsIÛW8ø¶cا22‚Oð8 LÚ‚EG\u]ùäì£6›`H)7”n-Æœ+=÷QöËÏêKoscséUaZuŒ§YQðA‹>ò¾ ” ¸(úÛd+är’oÀ6Ä.âtgãJ6ÆvÐù¿îò´xuÌýbØy32ƹêÞ«œ$¸ªdð"°$»‘úë!Á6Hîï™ÿ†À-ã1xŠ}a¸ÚEÛ^Ì©Z«å!<䪶Ùíéë–Ml•ë™/—Ëë¸ßl˶l˶¼üÂôûÞK˜wV“†.& =ÝÑíù’Ž.ç4]­ ŒÏl.‚}–†¦¢F‘ ¤úN P#Y#Ö€«º·ûw¨Ð¸eµ&+ó@€©üu¦ý(»Ü¤ ›‰M’bL\ ñyè²Ý$·gÔš¯F ÎÙxX@HáZ2“J ¥ËÃÐÎJN`u½Í/ÄOLk¥1w™üD z«u•@SÈì#w¡ÒÇGO46IPPÂÒüêÓý>eÇÈ”ã: 2­òg¸•µV¦ø.ùÍÈÚÏ·ÀNÅW€O ±D8—½ga¢r{Ñ4·’ Í;‘â¬ÿ<€ì#àÆØû•ô O1d øp M2YeÀÊs9æLP°„Cåòaÿ²%ÁËëGÁ=­G«Ð½À2Kéü…e#9OD€}Bû¹Léñá-º»Dç}ô„ŠÀ¡X¥dí‡à½µ| KDë9 ¼xB¯©¼¸ÏÊ5ËjµºHé|7±<†‰Mh® ªzïj`ȩ.CÏÏ©|»m1Í¡†Q iTŽH7¹XÞ8¦ÔÖò5å&/–”F†£)˜ÜEgØRÈ6³áº‡ÿΰûå:Úë·¤pÛådîþ"¶ Mø7£›9G70Q…a×^s}cךâL€Äwyæ—e³‹sÇ{8Q3Pq»g¨¬Íã+ú¹C¯6Ä©ú5?tyø§ùD]³¯µEqàábÎkÍå:ÔgRBm»²’!€ê:}4Õß ½×Ð}c,ž-Ì|qçÎQ7¯mÙ–mÙ–›,írù4Ú^ÆL—!Ðý½úìÖ==Ø£Õ¡UøÚ¦1´›à^(^ 8!«¤‹Ôõ£JÁ>°ïª!«ÜŸ-áZ¡[½ Ó*²[]1©Ýj¼ ˜-#³"ÖÍ9J–1 ‹9;K™\Ùíïu`Ä”5ssÕ þÞý£0^YWð$+¤Áœ¤Œ6DáHÞjI¾Ûßj§ý­À÷-í”å|üe<˜’îòZ/%hˆÑ1™Rƒ¶ ¤¨»&33røA@R gù5f9]ØRÇü>QVÃÿÛeu®iæšô³ËdcïªÀPÿ¬˜Ázúþô>´gå8,D ª¤úÂ)Ö®EÊï±Æ’Ùâw#…?j»l4Mè›O*+fvW«†ÌÌðà>‰_Ž5Mv·y ƒ¶A›ÄÐç2f€¹lsºÖã¤wå)™Éø$%£ƒ—> è±]\Ôf3§F]‰i5™ÑýÃcúx÷ˆÎ¹q=§–?u¼ÅkL  ³÷W±c©n!204(×ó¬~F¯©Ü¤µcì|í¯ë2ƒ¿e‹ÑR3NƘ)ÀЈR3Lêç ¥Æk›¦IóRDZ;<ŠĬ†ëIO§ÞuÊpšL‚oúyñ¸¡©}RU†ìê6ƒ–|µhÊ\—Aë²"§Zýa0Gòá¨Ç wÄ¿/ƒ•ëÛ¤½ÖàœHo3H^Å@Z;ú¨>©?%_(¨È¸@0Á®W)íCeL‘ߦŒ±ê{kd#ó‚&Mú_] [CÏ,žÑðÅ[\C)˜Ú|òsK"^I»XÕÏÄwÛôyÝÞ¡ï¯bïlúì«–¶mŸŸžž._Z…Û²-Û²-/PšÉêT¤yîbˆº7 ]ìïÑÓݹ\Ðáåœ&ê™Z1À°Y|¤Jga٬Π‹0 ®WF‰[؇öqWLõ ¢*ÆË1»Rÿ]¨úÞŠ..Åñi.n=×ì2b)pS i©¬ÄàÚP<äÞ|ø¹‹×÷×äxYéåPvGR~;ûà.‘ì ñ­Ä$aŠ b»$ÁœE ‚l 27±®í™é¿6C¥µ\eº¬¨²Fé&bàD¯Í¦º4††»Ò¨²î¼ŽX‘Lˆ¢.G Ø:¦(+Ø @Êñ ±{‹«®Û1h2˜ ŸEyÈQ@£Šh Ø}bïH–m¦„hÜq¬eÝ ¼åªRnOjÄ‘}7]= HU~.$Ç’CPÂ㦑¹ õ“AA†É'š±%!?ÃŒk€”sVgÏòý!`}ï© ¯°¸ŒEÔí…Ñtµô±•™÷6íŒÒÜZLfôùí7鋽.MW °Á”aeuà~ëYzÅ{àÖ§ºîå±j%~—Ó×f€»Qp¤iš³¶mG™4Â䨯ri©?sÃÙÄ r½¡ð…¨tÛÁzúƒ3Äȼltë4Q46K†Éã÷»r³B¦‹Æ‘@)’þëú,ýj“>½Ó´Ö" `ƒ°ÆŒ!P?mKêâð­Gºñ-»Û6Bü” Y[³ä Ÿ)¢¢¶ûFé›n9O¶ÓþòµÔ+kcŽ kWd©!˜'õµ5¸2TÖ‰a¥}¬î¡ëjfG àõÔÏC0æ5Ô:>£„󟊵åV:GµÚÍ}èA¦(ǘ‰q5ïÂ,0³jgèš¡k7¯Ó«¿ÿº¥išùÛo¿ý-Ž¼¼-Û²-ßær~v~zp¸ó°4¸:Í }|1™Ðü ¡ÙîÝ™/èèbÞnõòO;÷ó^ ç>ªøT9 x0~g ‰ )häA÷Kêž%²U¾ŽŒ zØp7°Z£]éÆÖg‹¹ÐPŽ“—Ã@9iÑ{="p ¤€5+àÊÏÿT§+Ò©^uq@Ȇ IDATtÍPv‘Åò3£Ç”( ñA ÜÊD§Lt!ı:H·±ÎÍÊ™=•-)¤°•ô2c“•Øέe¦,Šº½èõâ¬%¦ÌÉ1‹±Ëu²²RòÜYæWtmÝ‹ŒqBDk~¿Çàœ~­@;"Ê‘5èØxºAtÚšQp¢Ñ| hr °·Ò݈šïÉ;m÷Ó¡@t‡³£´ð4_? æeÄÌïÉõ„r ë}Âu+Ÿ§5hïÇyí£± ÞûŠ*®’Å5]ÔÉ$ÕkUƉΓt×ål—>¿ý}1Ý)SPp¹ÿxÃÉÛ‘›Þ³ãarêüËs%pÈ X]z=Çæ(µrz²3¡Xx/³Ü˜[ÍOúÓeŒ±ËfP³/j€b„À2ö÷¦4¾4¬\•ÑcSÜ‘!à&.«6%yHqGà`ÊeÈbí:3¢à9`V‘B‰+•&_pê§_újÄ)I!±mé†óv»ò¦m©¬Ý|þö2GÒXO&“~îG eÎt8ØM ábD¨b…© Ô*]8|SÜ  _CÉŠ9R XïØP¼ˆe‚u]Œ ±6.uÛk@Ÿë7uÆÄ?ÛyÕ—ìà#sc+ß{¨ `Vë‘€"Ý ™²’Ì+D–ÊUUý¯îÃú÷¡q¹ê9_§´m{òàÁƒ-8²-Û²-¯§ü]ÆH]6ƒb—«ö¤´^4>ßߥoÒãÃ}J1Jˆ ÑÉ Dc·=5w<ïÀr^?›áÇ^üÜ@bþ¶°ðœVhmýº2fi€©P•÷LaìüŽYÂ,Ü}”`Ò#ŽÀ{«»r)ræfÝ'⯄À‹¹€HÙ§.Ë©þŽ1­²´ò$8ç•”ÏÈ4Ž"É;‘è-¡˜Ü1’L•Ýj¢õ¡ƒU&e$àAçQTh#ËQ,Vˆä€­?„3Sç“Šâî; Ùcsv”žq|ñqƒ\TÂ@ΠSC†JÅ&P˜5‹OP)Jíu§É.5!»ØÔòl«*Fþ5~K¡œ»NÑdí:H__ˆÑô$òx`Òõ ‘ìÑZ¢ï´Ä=‡¸¯7ôàZåì‡p€ÆN 9¶  ˆLžXWQ;ØÉ®$ø˜vf±HTÏëY:ûßqM㢠›ó¾¹CÙ`&c.Kz/Œ3 Bº•µ!Ðùþ>}tçmú|ºës7ºLÍ8#† ·Ð>}y°\…â28vT ƒã¤Œ|ïç;c-Þd¹1p¤gʈùƒŽ¹½`¹Nì!—™ë{­Á—1¦È£ëŽ;Â+Y£ƒ`2Úa‡>—º`<à*\:Q\é)}·ðf@ ]Q[M¬îè‹Ý£ø°,×”)±Ð¾ s¹ÿS˜šð­ÌVS/U ¥‚ê程RݼkËG÷„ѲMA° Oº…°bô!%K¾!!´žZ)¯Ž±ßÇØ5 ²‰©0Æö¨Ÿ€G­Ü#P1´t×PÓô2‚ù7—"‚}¯‚ÌPŸYtôœ¶Û[€ˆy[– ÄK\ ËSƒacý3ôý‹7Èyòþûï¿6T[¶e[þq—nÿazØuBc«>7tË\L&tï`>9Î Ä$bx¯t Z>½ÝK}fr©º‹ dƒùN•½—«[U|2¦Ù Dì’1 ¨(Ûè_¡;ƒu¢»WïL¹–Bõì'.+P`@JnC¨úKªL…bu,UŽëÚ–c[Ø9m±V2H’\m¾›\2ZŠG¹…8¸Ù±ˆkñI,õ²jŸâ $í¿ @Š©Üpæçû”‘a†QK»X¤N“θ€4€>ªå[sÅ!‡ÂÌ®fÅWA–˜µ†¨³;)N¨€hbÇ|É}ÀšÇÙ*Ú¨ýa±HÔÕ~ï\]Ž…äÝ–è¸g`1×0‘ü” 8© 1Ç«qð+»õd—ëã0¬+½º9J‰3„ ‡4ƒcmôco¯ÕD&úÄõï$ú> ›èûšNÉyþ³MÌÝL¦ôðè6ýòöÛôh:6Zf_ëÔŽb‰&º«¸Žn:Œ° þ‰×À¥np,“å|õÉ¿ù_þÍœ^S¹Qk?3?èPÕ*àjÍÌË.³ézüžFÀŒ±kêïjÆÈPF›ºxM;áUŒaN¡…hÝ®ïø?nf>דX,n*•S]HŠ°E.®©Y©ÎÄi:†R1.^_ÔÍe2å±TÄxMùïýGžÌni{ÙEÇ/1F´¤±,ë¥B‡0t6 ænR¿ÏžjÉìùØqË‚€X¶JtÊõº<6T†”|,5cdlx‘z‡ê»êÞš‰r`rÝ÷ær­ñúA’+âňMTTÅXƒÀV õëÌ/² Ï4ÔÇ]B߻S÷Á ‚×.Ò#~g¯½!Û²-Ûò»0ÿ®—±û“u˜n'—œ¦¡Ëý†NfS:ž/è ¥NrTë›;îÏvZÔkHqPå)ªh´²× 2UçŒ1¨ºç+Șo€ïý¼4EŒ dZ¨‚têëu^(‹(ŽgT±x XäÊ‚A¿ÍòÃT'= ‡ÖÞ‹Ë vÆwJ¿fù·iªFTšœÝf¯%^¢ &>’Ev«(ÜàJ£Ÿý-½/>ÓeLŽÄÒЖîW¢ …lPÑ¿bYs”éãΈ‘“…L- Le†UèFÖ‰fÜ‘Œ©LÌ°V´ÝÊĶ”Òy¾…+`>y Zu[bÈÎ㳣는æ-!z3Ïœ¥¡Iˆ5;‚5áÙ;8ãÂeþ5ùQ6”"KB—OÒüì6z¬HT\§ÒáH! Â°FÄ÷Û«@?´6±g›²7Š…«y L³]ºwxL÷wviióÇfK¡×bö R<{sß“ú׶£D›;¸yê|Ãî’N­¦Guw¾Êr£àHŒñYÓtA/ÚAfS€Õ! dÌ…ž;üuŒ­rÝ&ƒŸ §yéaŒp³î N4û+<ì¢å«py©h¥¬>õñPOàHèÀ¿ž@P("Y”Z+e·Ð3q3 !p¬!ÏoXcÙ¸rd‹}Ê°1Ø!&6"*0 KxÀ»ujÊ~….Ì‘¡otùoÜv¿R©ï1àK b 1Aj&I­üãgCõÔ¥.Ç™'%X¦I¬é§–Á«t4ü ÆUÖ+6ÿ_gCq‹Ñ0ák™ã\dÑ~¶Ç&Ðé5%顯-ÅÚ¶l˶l õ{ûɳ™‰¦¦Ð­Ëõß”éøg] à=ÚÛ™ÒÁrEó%ÍÚH“ØH@¨6×C@Ö|¹> ±DPQÄX#ÛĤ8)ù€÷GEJͪD¥ÝdLr¥ÈªVe¯Y»×£ˆ¾ Û’ ¨²»Æô£T®'°” $Xý]–Qð$Ç51ã‚Ë’S©ªÆÙ+•bü°w:À')ß3!:ÌA[“Y`N$K6&ƒ+ÖPPÙ×Ø äé`Õò¨sýg˜.×®ê›CÅœ°âAÉ&ºT2fn`ÀÑÕ±pÁ À‚é\^¢¥.&ªTªÖHЀ¢Á¼:'` òlàLþè@HÞâ;‘xê.W~_žÙÜ.\~÷B †’ö9á\ Á×&4Zr'À‚À•¾‹ö^e_,R¬ ×?¼ÎBG0‡Äƒßú긞¨mz¶»O_ޢdz< Ê6¸'ÆHó>vÚ”3ÒtKù½ØKdÖZ‡êÐîÉ쓵 _a¹é8I€nCè].jðcSö-CYiD®¢Ð0R×[;x'/¹¡"çŒÉšQ46­úĹ%$o ñ^pÎç¡‚>%¥¨Ó¶ 0ª: ´Tçu–ù3²â:‹uC¥poã*±G–·B§Ãùgã§ßW›Ò—`»ÒL=…VÇD‹fJéÞÞa·ÿ•2§»¾mrëÓÞŠ°n ÿL2ºº^ö7ÐEËíczåFÁ9ï1VšÑH–™¡rhò¢AV¯J!<æ²3VwÍfIIÌZáÅš;*f®€Ã¸ÎJâÓ¶¯ mcaá.A¥æ+UøiÃ+š°Ð"çŸZw,±Õæ%º]”›†à剤 ‹|óÂ`œ™«ç#·Ü2lZFQ+(’ŽÎªpChI©…4®¸¨ÃëþJ? ›m„õi_•«Ö–a1`y¶ÉשŸèj€`èy‘ši"Ž”²ÙÀ:˃Ü㈶?/”V¸¤ßVktº(qµ¸¬ûg2ôÞcß½Få’™·n5Û²-ÛòZK\…‡M豚bgÊÚо®gÛŽrÓ¼ 4Φ íÄH‹.–4[®(¸4ošpyÞWÊJ¡5‚ˇ~kF–R¦àêHû*2&ßHÑ:m­žsX…ÓêA»©dšk[ †à™[ Ì _¹ì… UÀ‹2•4Â8Œ*(þ/}öw÷!hƒ+Ìvþ¦ ›) ðQ<$âKé2ÝÈõnY6?²ÜÌë ­+À}jas…)\IÄuh¡bέÉáè J0ŽÂ à†º«”ê¶+Óꊒ™2ŧÀ!’ªn‘an+È㓦¿'úzR6 ,€ÕÄ=Ë$Åy3v±a’O3rtÎlX—â×¹aSȇ©ÊêËžÈSôúZY«:ß­atP5²h€ÜZ¤…yT>]ÄDÐöXIýk ü™p½Ù.}yp‹ììÒ"qlyá> AŽáÀ®^Å0Û`K°y‚³1äê­R÷iÉôŒ[ùÃGf³YR Ò¿£«2Á •M@ÅXŒ’:Õ.>³V„¯J<ôýPêá.2v¤ebhvÝËŠ‰ “,å¦V6gHétÎéâ3ÊÏišØûd¶C ‘¡2tÀxÌfÛ(0:wqª'×|À+-CÀ \PbD2ÝúŽTYýØY!ØUJšiŸj]p)F,CùpHݽý¨×Ô&°b [¿§B%R³E RU×ë¯]ðY›@€8à0IR°fTÔOV\=¯fÀt×t4Êü™j T2Ä$ÑçÆ”¦¦]\Öm‡¡>zW,¯‰er~xxxñªº-Û²-Û‚e¶;.ËÙ)ãþ˜ãMù¾Š2QŸ‰\X†@«&ÐEÓÐÓ)-Wt|¹ Å²<[ Aný¬#ü¤w«±É Þ`‘hÃÍÚÏב1Kë8—Uìå;Ô U×#ª$Cæ¢=uÚ{#ûb¨†è@ˆ³‘¹0@€ŒïQElÞ·ƒ¹ŒAB`Õïã”äÑ®´Y—s¦Jx ñ9f19tí ñ‚)¦4ÀçL¼èÛ­·3SÅX§3ÃL I*&‘1I ë CÆÉn¬óÄÐg•.?’]HrØ<˜Ñ%È✈®-­FÁ›úì4«‰Æ4ÔN(@Šçä†I« ·Í¥]!º‰îÉ ˆ¨ûX<‘hÒY7¾ðƒÿIfU ŸçV¨Ž¸Wî÷ÃnÇWU"À·¦-§D% $9™Nº 7w.4]önÄ„.=E¤>ÏɵÿâË+ÓÀù_d·¨Îš¯(cï@IÔÄA¥†Îs<ãümPTµ×*ÎýçÀÁYÁB­ßaÖfY› sŠ1Oâ_Qñ@~¶³ı hÂw^‹¢ §°ð›$E"dŠÏR¦›„ä![ÕÓ²2ë;qx–\Ædˆ…ÖjJÛ ŒØëæ†6Ð7é_O† óQz´ÚšÜÇS![?÷×$àe2¥F2¨AÆÄñÕžŠ`úZÓ÷+‘1SÎõ¦PûNìƒâBà_«9ƒ Êìi‚F>Ä5†`ZîÇÀjÄê@“”MÅR<ƒŒÞ3EÈzœMÇr7´§t ÛlNRN½ âfÇ~*‚û½61ê1pÔÙ/:È@QfGÛLèÁí7è“Ý}Zpp«yÁUP<«•í äΠŸÍEÿ¢ž$ÕÚÁùÀ¾ ¦Ob¸ÿZ p7Êyï½÷¢ˆœn3ðßØw4 ànr©+W/RÇZÌع®D=Äú}ÂØá)ÒÊ×­Î-€5sCŠyUrk )wÌ‘Æ(ˆl›“¶É¨›¶(¬ DÅ5ëVð¸â¸<ý&hKÈJ˼ \KŸs˱D§|Ã`FŽg¸`¢ôTFDU‘ñ(•Ìƒ§°žDßãßœ² ½¶¼I¤J¯Ù Z®ãÖBÕ<]™‚õ1%61( Y³†.¶Q(TÄkõ­ãzi;×-Š6–vPø¡^<+a¥«³W?üÿdùèZ±9®Ë¦ù&¹øÙÏ~¶‘É´-Û²-ÛrÓåoþæoV$òÐöü‚^îg¯íóvîº#ÆôuŸ3Óýݺ{|H_Ò|61‹7*v< £©BSà•åUÛS0Sðì{ 2&¾Ú ›QªïØÏ¿@Ÿú¸dýµŒqÖÍ™ ”&³UUxYª<>ew·±K1V¤öo`Ö¥g`0‹Yðõ´}ØËn£÷›VlB¢c)”CL±I’bÿýHt[(Nú6Eí3Õ}³K‰óÓÜO¤g`DÇáü“ "c̬٦FØæqÓc5½"ê[1]fŒ¦Ž°ò›Ü“1‹&ÿãÌZ1#ª²rº†VÀG\×Áš±‹(ìñ»‘Â÷[ »=ÛCYD}N983A¨wƒ ÆXý¯¹!bZe2ð¡Ÿ'z}(‚ü>ª£Ylñ¹1‹a„9Z°UUÈì’Ô®Ži‘ýÑ8®Dp&|p!ŸÚ²Ç¯™Ïvé“;oч»4×Þ+ÜàÊL°¡DV=Dç¶'é}[£Qn6}÷1ôN.Ð}T(Ùï?þýǯ5€åMg«‘¦iž¬Vý;^'ÞHívCCÖý‘û7ÅÙÄzþ˜]»ýäkÓìY…6'Ð*¶jÛ ýóhƸùëá×ÑÂ`¹29®ˆš’ˆ¿$ Çb!øæ+~ H)UÈÚÿÝà{Œ!ÆÓpùø?M/?øµ´—¯w2ƒM=®®åð)ø7ZuZ½Bxgè6ÃÅ$¨hxz¼ráË["Þ¶iÇökkÌC E fÔ Æˆò"uâ½CìajËB]Æ€œ\OìÙƒþðaÖÊ£¤†T¸êy˜…4•öYê^¶—Oþnõð?ÿ'jŸÏ¯ t|Ó,mÛEß2G¶e[¶åµ–´ÿwôßÒíý"¡ŽHÅÞºnpAÞÏ[»ãT Aúû20]6=“äÖrEo\Ìišb’hHçÀóAw ¿Ò¯»RH_GÆì~—ŠÅ0`ܨã# É›w Û¤à‚[ƒ­nŒ1™oÛœ§ÌÒßÑYûe”Âà\\K¤is½eI1W·€ÅÓ¯ê1ïâš²M2À‚ Ù ËØ©Þ6g°éX ]×>0Ÿ’çD´êÑ óãÓ–ù~’½¿ÍcLVdKa¬zq›A ±¹½hºáÞ ûkXÙ#äïlUæ1ÞHÌ`Šþæ.æsFSÌ2Ìqc“Á(Ø…ùcï|Þ#Šßi‰oÇ.š¡%„°‰Byû'w.9QÁPžŠ©PÈ¡ù ¥l»‹a/#lˆ’Ù„7éÃ4ˆd–DîV@£g1vMÑÄÜŸ™1”&ƒa-­hV_}žÇ)åËÎ}rë6=œÎ @ÂÖb$¸ßç\uÁh»|¢2šŽ€Áu-v¯¿zϲý `¬®¾6þþýŸ¼ÿ Ž$Aú¹\T$›€«RõÅr‡3°Œ€£©{ÇX+â63ÙŠE©à†!Íz°èô3À"ZèIµ9ä¶`€_nÀ²ÒÃÓc:]6®h•kð"XüH ÚLݺÿC˜Ú%/Î?œœßý|ùé£×©Ú)[„*†]1nƒ€YÞ{l›¨\hŠ`$('0k¹Ö hç¤wXºÉ«®b…Œ5kdˆù1ô¼MíÂ2Ô®! eèžú™úwX>ü­–'qrð§Ô̾'öÿ`èÓšö‰`”Y´ªú;kX¨S´©¥¨;¶clWOäôË¿mŸþú#Šó+7özý~ŠˆÄ³oMƒ·e[¶å·¤³úkîǘ¼š/nòŒ1-*¬ îýz.‹ ÿV ,woÙzfô,¹Û,–tûbN;Ë°&€6¾vîVìˆB£3ýÞä¸ì«È˜CñF\5ò@¨F±¹Aã@vR ª¨­R(ÍbÝÿ¥A)ñeCP—e0"Ú»"ôVö˜´æ~”[Q:µŠ‚º"kl_ êö`Š\Ï¡SáY‚`°ªç÷W`$=¯é€‘ôîäbÎ1#Ä]£`!é¼gÈD£ó¿1—tÕ4¸Ýr)~'¸ºöF?fýX*0D9p­­Ë|SŒ:ýÅÜ9Tö˳‚¬x$<7¯*ø|ͨ‡ ¿F.©AÄ ”uSOÁ=õiÊëÕ™[ {X•ßô3]¶Ÿ¤ïB “ýCúðè67¡˜ 芠amóÌH À†÷QtK5 j_çk=x­TcI›˜ˆ¬„ä˵EõŠËƒ#É?={a(Øê&åõªÏ†@I†Êu²×\'EjŒÝ_DæU“§f­!À‚‹£Bö!˜O<üÀ¢.(ùÏ62=¿<¢ß?y“~w²OóXFÁõ— IDATÞ¶¥f ŠèVa vå\›î¡ÂÒ.¹½¸.îýb2ÿüSY.®ì ,cAWiPËpTôkŲ¡Þ¶!8HB°‘;úÏ @ñ K« «R=¹ìw·Æè]×éÅ! £þ®¾®v?†€’úïô¯ištöïIØ{Híé\dÙB¥CmÀ6Žµeì=¬ÞÕÙÙ„Ï+Kþ€šã·Úé­?“°÷#¢éq?4Ù„ÒßYö·nð,T,\;JêHÄhQ‹«v~ñy<¹û³ÕÉŸ3—ïzÌ·¥0sc<ÿÖ5|[¶e[þ@‹Üe+"™ ú¦¼Ûv[î»Æ¸°X%~¹~ˆ_k€ˆ¥,Me=ÚÝ¡“Ù”îÌt”²Û¬Úu—rÅCÏ©þ&oQ™çR¡~“аƒxL¡°0¢2ô]á~  2£¾Ì1'*O~d~±OìzW CVŠCp¦ƒe¦C“1Ê1ÖŒ6¶…g?dt)•Ä ¤änÊ~G²v#ÈÖ³G|:%9µÖgº ·R `&:eâó ôˆf}“)Û¬øe˜¨’Lþ,7¢±DV؈”étM»•ŽB"eLO+Cš“E¸¯Óú ªŠ&v„9…Ì]Ân Rê)é˜"ôvÎL“kôñd Fš>Su[Áœ]û4ð®|\°œlÞ9J@Î obßÇŽÁ*3JÀþ©,Õë00B]̬œÁáŒ/×,5 §’•½ˆÍ”Þ¢Ï÷èR3!„ݘ«Û‹Æ,XKÅe_Ôz¤††ÖeZæXÑ «öžªc³ÂC´”–^k_zàÈjµz6N"2 ˆY§Ícn\'>H ¸¼@r˜$øÞÓF¾Xµáñ$ÈKÜ!î O¤–jq«t¡{ÎuEXŽ Âåƒâ—ËfùÅó7¯ßž<šOÌÓÓ7lŸ˜Šð°ƒ4qv\„”¬xþ,,Ÿþnrñù¯hqÿÙëRíꀫu¹j¼iCpVæÖ2ÏÕ–ÛØ}KW ÎÊÖïvvãæK~ RIÉÑi¸Œy€ 3Tj ƒªMtÀec]C@ º¿Ôe—yr›øö1ý³Ãfò£KiÜ_­>ºÛ¶_{&Ã~ß 'ÁCxåþžv/N>ˆOûóöâÁ3F6õãWyÇoHI`èÉ·±áÛ²-Ûò‡WæÏ[‰"´»æêˆit!KÍð™SÊUŠh©Y#n(q "¥¾¿·Û$G‹.4[¶RL3d² t~aL-²vÁ×B}US¨y”2—ä(`tÃñÆ—ì‚ûsOƒÁ§Ì(¤Ïuƒ ÖÞA*žæà–Tk TÆ>ûô“ú0(ip—ëZDYr½:ôb”u±±³¶“YÎÛ”.æP(P’ðõ.7uU‘.C Äq·„H5÷EÌ=Ö3FÔåÆЭ|M?_u½1¸%Crÿ$D³k¿ È>—¹äJ´JÖ óå,QE½É`ÈÛB”À‘¦?*˜ :“Àh·–iØTzŸv•]%ÍdwØ:Rà ]¥¬dÐJ簺ŃÍÙ÷œÛ,¾ÖpÞ‘#„^§OêÂ+msìÓOàß|ºC÷Žè˽Zá’c.Yv`¨0´ð9Ð僧úB÷–8v¸ƒL9›m¶¼¹z¼œÓîô zÍåÆÁ‘ÝÝÝ‹ÕjuIDûúÙX ¡2–åæe(×a…Œ$õg—móìÁÅοÛ_®>ÙŸ.ÿÉ,´o4üÿ³÷fM–WšØ9wË­ö¾’›è…=­–f$Í gždÝfûA?@zÔЛˆ¡Çéç¦þ‚L|hõHÝM"›  €BíY¹Þ-"Üeágq¸7³€B" ¸VÈÌ{cñp÷p÷óùw¾ƒD—¥rM2ѤI&Óvb#0¦ ¦°¸oqv|jïÞ;üá£C3>²õ+Ù5h_éqKa«2² ºŠJˆ‡2Wϲêôsœ?üM6ÿä°å× Ö'´ vž¾µør-4çô×õ­:Âü$Aª2© Å P‰B…h!PTØC %ò¶··a±X4Eô:>ëúo²Šq±ê¼hrI®“Ÿ;‡»Y±õzž½öj>ü£â-/w³mŠ½m“¿ö²u÷ëêÓÏëêÞÜáQ]ÎÛ%@§|} —NõdÍI¯añžãƒ¬>93ü]m¶_€|ë5‡Ã‹ƒfì³¾gãDRŠb³›i¦ªŽí郮Ž>ú«O=0².¬)ÝÕ| çÛÛÛæÈÆ6¶±Ka®Z8ÌÍÕNÈŠf>’'.Ÿ’ðJ=ô ¨ƒæ<Åðð÷šgÌÇ Ø^V0Y–0®*È«º Wè°EdKÊÐ/ÿb㜕R>å3 —QN³0NÔZ%972]ÇŽ"´Ð¥ĉ$àâý¢¾¹4G…K¨AÚ`ÒLÉ&vIin§{tŸ6&53ˆú‡ã!æ„pPÝaì*·žœkËàK4&®M¼’‚[‚OÀa)Ä€q jA`”0*íoÃsõ­Ù0&dݳ@Ž¹„Y`€ ÿ¦”¼jÔY«wKµLwsHÿ2´€×àZÝh²8Z0sgçXaéð%¤}\k«$Œ ž wD4ò;=qÕžŽã%ŽÓÁ€êö”½3¢9RÔõ¾ó&´Q}XO… F ‰¸1hu:7ÀÈãÁ¨F4 ‹ä“„ÇuT(áN@ãAÀåÞëtsø™£f“‹¥-¢G8v‰ZvÕqV”ð5ÛWŽlooÏOñÚy×¾P²uYFβ³®ó4YoV”ÁZs4«G¿>®ŠOǦ¾¹•×/Õ‹9ÔW›DT]Óh½¬‹•È¡Ro‰µ°˜WøðáþöÁ‰ùôp^ ÛÇm8ù]åÆ·*³õj Û5f{à Ð÷„œ!oõÞúAØY°‹‡föèƒ|y÷cSÙg úELdg‰úöõ³¾~×ßM´» å¼º&ƒŸZ]t†Û9FÙÀê rÊS8„] ƒæŸF꺆år eYö>ï:Fßeø¡¿×Ç¥ :vǘÁ YvëU3xçÊ x-óÑ¢jíâûþ0Ã_Ê·nÚâdîÜãÇuyßÖŽk»ÿ¬g”ØôººÚ68¼á²­\6¾ _AôÚèôÜÀ (H¹FÀÚåâ¾=ùüWõÉ>ÂzºèP§{X:Ï3(¢lzxx¸¸4¥ÙØÆ6ö­¶e½<ÌŠ­Gî ˆ¦^š/ºLÛîzªçsPJô;;¦*ÓC_èŠÿ|i ì “d\W P2*Ky]'g@´w/"h$§ã%p¡çÒ5¦£vì€Ha¤¤§ÅŽpßÕëB‡ÝðzGDä|RØϭɃ©í;vÂé QKð: ãЕÍÝvÄIì¯,j èj 0¢ )>/õ }A=§çM˜+^R]þŸ×%ñâ­%ø™vŠà¦ál²›LìeÕY,õCe¥'b5ÅèÇ èC†ù 3: /”®aº&Í«fÍ@Òæ=ÏpÕú çŒ@5n¼$–÷¹ùÕ÷ ‹Þ¢8Ä ¹ƒðû¹BUØfr{zº³A¤¥¹?ª%¤ô¡¨ãÈ9;Do^'ÈŒ‹²)q%4á|G“-¸»½ÇYÞ _*MCLÛ€:‡¿)}CÝ_¥VºãƒPNܘθ[ŸÀÝ^Ë8fÚ„ã÷íÑô¾fûÊÁ‘O?ýt±»»ÛyгVSF@Ÿ­b<­ÖÈ*û¢ ÉÜâñÒ N¦u}wXW&yuk«¨^ÏM} Á nSÄÕ…TNN¿Aê&mg·U Ožeüá?>™âᢶU{`ªƒ‘™VöôŽ…Á^“«lëÕÚæ7-âz7»€úþà¬qöÄ,žüÎÌîý>«ö¹z^~é)¨]½¶H*滊Q´ÊúÎO?S™ñ“Ð#àÁ»ƒð¢ .ÝÆ’6E¦$FÃiyfì Ô`kE Ü×…ÿ—eƒ%°HXggsVÈM‘eÛoƒïÜÌß\E&E:Ö9WsD–ÜÛÜÝ6ƒW^rîäÔÚý'¶úüÀÖ÷î×õ“So¸¦ü} ‘U DÿuÊ¥©«}kOž˜ìøSƒƒ›mÝÄl|Úы|Täšâ‹M¸UøÛaY-?°G¿og]”}÷^WoÏ9Hr´··÷µæŸÿ†ÛëðoàßÀàÍo{…llcël±·˜N¦“‡ñk^P-Ä•£‚Ñ/¼ ×[ÅÊiÖŽÎÊ­“8Œ¥2G&‡™Ÿ¿í¶–%l—-£Äè¸{^n¨²èmÿAíÜ«OÒ5&ÍÌ.Dߨï¹úºÏ˜fzKñí¼Ê%¤˜å‚ÑiÊ©v­\_"õz CšØöÏla1I$®QŽ£X6é3*êI'L‹uãªc†vBÌ6&w›ÊÖk“رS×mØÍÌÓÔ°RO·A^cJáv 7&UË¿ÝG¸cæ 9ðT’ôšba5÷ôk'ýà¾(Þºn®:h¶ŒEêPÁXt¼8Õõ¨ÛÀ)`F4Hº#³¹£óu :-ä¸ô8™HÒ|iÕ6¦^þ‹˜.÷¡èP õÕjêXf’Èåô…UyÊ¢€“mx0™ÀÜäòÎèw=ú[]SUÐÚ6åòàd¥X!¿‹(ÅÆc6›ŒZ¢·J¥Ü6çàîîxw¶²0d_98ò£ýÈþâ¿8ñÜYi` q–hjŸÃÜgëÄ\Ÿ˜’”Ù-,ÎÎfËâÑÉ?š ªÛyõæ8¯ß@ô:­œœuTéÝÐVµ=><ÍÞ¿s€ŸÌp¶ôä»nÇöe00?õÿ2œ>Ú£ßW0¹¾€Éë%ŒÞ0ãæšÍ³1r<7Õô³l~ï7fq÷®q‹Å×ÉÑL‘UâªçaŽ¬û.Z|f%´5RK ür¾»k$žè’´lÑ ÅsjùŽx¡ô•C×ýÈ IQ PâŸ%©õ±?ÈV}ÇÒç9€y-+n¼5þénf^GŸ|M‰99^ÄDËMŽèE„Á͵"Ë®ìAñJYWÓ×½s¿ª>ú´\Þ?XÔ+ë^6URÞ¾°m6ˆ]̳ÂÍ×9[uvõ›o½fp ‹öz¦Ý3röÄ?þ¥=ùà7ny2]?ôÛóαÖýþ÷¿¾ârY †¼ñm¯mì©ìgPá@ƒÏiˆ1vxó4¸…Ê_9T'¯ ÿEÄNãjOy÷º$ /Þ:ÀVYÁ•ùÆ$¡²+6¯vdÈ—"''rDÒ°T%rÚYcÆžœúX]Cm E‚›ÿ0¼XSùßÊõ–ÌôPô,JŒ ³>¢{ðÆhÝrÞ´3§Cv¤m öáUßH–a!d†„A5ØÔéhƒ¦†Ò#qzÃ+˜Ñ;êÄFéˆhÝh /5Ü –ÂN«úå|Ø×'Y†•;;ôNÄSg¼¹f‰2vè…µBõHÏFÇXÊò‚ÄV·ýëEhŽØ&¾R ±àƒx±Ûv/XÀm˜kV¶0 LÚOƒV¢´‰!– £ME~ö‡‡Ö¤Ñ]@QÝ’uWØ"& )Îu×ìÒ‡õ`˜…Îø Femû¥éôc§4ԘÑ>ßÚ†'ƒ!”ºSEcšFPeÒ`•“þmÞ!e°Ñ:AN,í1ˆŠðk®µŒ¨êybæž$!i/eï=ªŸÔvÁvÙj¼Ãv”e™«ëºûF;k÷ÈêªßÏr Ïk_„A¢­tX–5–‹ùðäØ îlÕû{ƒå;CS½i Œû6ÉãÜ$.”³…ûøÎ~öþgÇæ^]cU[ëºgw ]µôÁ¹œæöðÓÒl½_áÞ÷nø&8;Fg}&š‡ÙìÑ/³ÅGÀj>÷Ú™_[’,4«Ò:÷iŒœÕV붠Pjgà “À®z\…dÂ#&‰SnºËÐS ÞÒ‘p›@1 'XðeªW>Õ…IèŸ#)ä†Ê—/0Ö#éïÞ¶Àß Þ|%üpdð:4º’æ8Ú)‰ö$Ô¤BuäšÙe44ÙhÙÕ+&ÿÎëÅàþ£ªúíÊò³¶:uªüg•Uÿ­Á’U!jm qé¦`éÁ},‡¿uÙÎmW\ùŽÃ᫈8tey·>üê¦ßź¬úÀÈuö `Œ4æœ{òóŸÿ¼:3ic½æÁ³C6¶±ghîA µ×¹J7!#gÕ<ÔËB„0w¯¯9/N¥›¥õ¿8$P»¡‘¶‰¹„r8€“"‡’Ìæ0^J`v:R_N9þ±cüà $@ì|«*éTSô =s=©cšk-Öž¸rØ÷£$ v61ˆ‰Éð'`ˆjŠdó'Þgê> µÕ•Oœøƒªž0¢Á°Jä@œu¹§ pNe[A Cà´²âl·çêí7Òüm’,Cp¾¬™˜+m–sŒ`A7&¤íp /(Oõ•[¥¦Ž IÐŽ³ûH?6ÄQŽ0c!ôüºaèóÁ^õ¢«`äš÷ALB×1®{®SU00”l_Æý¥Ç5¤C¬‡˜FD©dÙùWº1ŒwqÓê0éKÜÿ-­5¥ïÓß‚ :.‡f};'ÂÂü^pÿtP› öwváîh N³Œ³aN½ ÈA=ÌÆ›ˆ´Ý‡Qa“†!à9a’1'vvP²©u|s¼ºgSF‹Ÿý‡ÿñ?,þöø[ø:íBÀD<²!1õ*ÆÆyÒç®R]Å>9+›Íy>?Ëžæ<çœ-kXÔùÓ2{°›WìËw‡Yù: $3LâðYpe ûŽÍÿ÷áóá¬ÂE[ŸOïlù2 ”‹=¸SàôÁw>XÖø]œ›ù`ññ?cypâ\}>Äå+0Šh[×V}¬ô‡>0mm˜–su:KÔÚ0"“£¡Åt‚ës‚iuÖtqüN1øî Åð‡â®ƒh'…Ÿ/¡³j˜?Z0Þå q¼…Ù“ܼz;ËÚú·¨–>ÛM”!å¼@ƒ^üžG¤Uuó)TóŒÝÿĚݛ`–7–GO>vËýStçcQ}õFÑc±oŹ±U¶C6¶±¯Øl…÷dXJ²–¨9•Áµ!¡Æ;›ÊáBv¹špQ¨åìüpÈ&÷‡­ËT‘¹ØïüîŠ6ÃMÃ$YÀx¾lœ×HÜQ/'è#M2QÎ%e§×a>ÐÙhqјÒe‘@´É¡À~6¡•NâÇ]+ˆjß:˜¨ *ãŠÊ^¢"qè tÀPe¦²Dì„Ø1Uþ¥ƒÀ1V(^þµW¥ì4 a̱ÄN"ÐÃ’ÃMu좵µpB& ßð*€Ý±àŽðÈK«µ•jK€û89¢¦­ÓÄj+†‘m Þ»e¡=?Ã8³<¿€xµK€|9o9À›\®’P­‘Ž¿f-ILhµ€Dѵch“@çZÁ^ý~û†’ò—¯©—»T Êë×Ç"ƒ7Ž“¶ÉËÉ}3M…º§(;Öáh¿\ÜÛÙƒ»ÃQ#ž  ¨~±t%(†^#œGÀ ½ûüŠ ü)ëó~9½HÙp¸áuYÒ÷;œãˆJµPÙ >¿ kÌ G¬µ§Y–Õu]›/ÃÂèËT“~¿*ô¢ï˜ó0NR[ô´Ïå™$ëⳃª¸weP½´SÌÿh˜Ù·ŒWÞF”}ÄËòtŸ|²o~q÷È<¬<¯á €"}†nYæîñ¬<¸‹Ëš³V¶.=ïª0˜”iÔ—¦w]Úè>Úé«NéfZxK­;„î©Y É…0¢:OÒ$|û PžÃ»N™6þßp8lþHBú$ '°žì0.Éãߘüp8úó+¦x× ¸ÐQ…@„.Ë5â‰r–:@ýÙ²FYöÂõlðÃ7ªòýXœþSBaÓ¿U;}}!C}çêãɬ­kt÷ÜÉ“{Ý银ÿ¦[èW§ßŠ‡ýâ¶C6¶± ¶ ݾ\ú|â$,$åf¿ÅZdéfˆAô\ß.øc‘ÿÏ×£,r}ZKTàà`PÀá €ÝÑö%LæËfQˆºâ­°ãüg¼îg`Sw–U 9Ѣ麜N9J´)äÔŽ|¬pÝéà„$ú¨ KÉDÂÍÃ;ÚHþ\`\ô9ht-qéJ1ð"œCÀÉ9ôôæ‚$›[†‚¡v£kb[P(‡Ókö{k%K¡4£Áa6dŒ°€«ZSùÏ s ÀîúPx„`g²^…ÀÁÐ~æaleBJZ O]³bœ´©¿­¡Õ_§Ew£e–`a€/:p{uÓçS€ÃEº,²&túÎêÃFš­ã:ëp`€C€~XH¥ú½[¥v€?ˆ¡’樖0Òtx="ªWWÄb ûÐý˜~ða9Âg;Wàþ`¨| :åÙlwM«AN¢Á0“Dmf¶ÏD`dm¬ÚøU› ”8B”OC¬%n'íáYH)•)ÇùìLáØWŽüä'?q?þñOœ“@‘>!ÌÔÖëBdÎÃ@écžœð8ïg絺†êñ,ÿä`¾õÙ•aõÊN±üÞhP¿æ¬—5œ<9Í~ùÑ}øõ“9L¿*èÝê‹0­-Ò×/Viˆ¬ê?šaÔZÕ×ÖPÒd‡Áh0Yå6OQÄ{¶ZŒÈ% (tÖÉ´à‚À¢Â4u~ë<@âuIü äÓ{dUXMº‹æçÞ·‹âÆ[Åð/ÇYþ::gh &¨1•_®³8ásh‚lSw©2ÄH·ÿ«võ‚ó©±Mé\o_MËÜ÷Ýyÿ^wlzo 0ísûŒBGç8ôÛd0dcûÍëÚýß³_<ôa5â#è,(2o¥{å(Œ_”ÎÖ'MT]êº:&ÞPÀ€B˜U)ŒI.%ÿØ€$ÇžI2ÂÕ&ܦS+z<À'*Ð!vÊ#°"Ý0è‹už=Ïš¸ô%˜QqGbqZ©iA…áðYì(‘³î"ñÒ¨C¹šð&ë´g1I´g)®ƒPN&¥Î ekñépm" š€a HØ ‘B›èA­µ 0%œÒ͈W’(OÍ`‹®gÝõB—¯Ï惛aZ]/ÚzŒ`NêJ“Ø–÷Ô®2 WYl5a9¦ ·‰ú5BÒgbm Е#À®xÑú8ì6m° ìé ˜kPØ!]€ýfŠˆ™EŒ—–E¿¦5`³j½IuËK74̲±$°ªY ‰ÐõÀì"bœX‹¸Œ z¬^GC³˜˜•žéx Ÿì\ƒ¢P@oœ! T?#ÇÂEãPÕ@O[$Hë˜qÆm‚Òñš-±…µ¬êŘø; p¢MnÞûÊdî^§²¾»æHUUó<Ï}Rªá:}ˆu`ÉÓØÓ0?Vݯ/ γ,cjµCûx^|r°È?ÛÖ·œq¯><5÷>;4w祭žîjχu@‰!0ëXBëôHV·ê\o®Ž"ÄT$.$“Y»ùA¬uJÄÑk+WÀ¹ÐÌš‰˜F¥6Ôã‹6¨~>Êø³½½ÍLo]zòZ–½üÝbø¯‡Y~ @&öñ¬ ßZœÍ‰X¢š<#¶†Sõ(K!PÌÂÖÍÄo]r,ípâ<`ELî~ž2BÒ:yÚžo-¦Óéô›ô@_À6`ÈÆ6vɬ.Ýaž{õØnJFŽ‚ãÍ~vP 2Ç,NÙÉÖN"FÎBF>×áz€ÚuwV˜éüƒúÿ‡¿ÍBrXäpœoÁnYÂÕÙFËRâô©ØÎE D“QBƒÀÓ-ÿN»æô¬(‰Ó¨–4¶+öJ.Zg…Ù Öhv|Ä’%‰JG‹-×…Ä÷Á½ù"í¥6ª'swFú(ƒ'šÉa° +!—¹qÀ-DÙgÈi÷›`™A›É²¶o‚|¤Ð—ô;Vðˆš¾)w^'Z£Ä!¦vÈ`â¶ÔS xbO\Ù¶3F”çê?ñØJmS­ê•…6ÉL÷Y T´—-À5 PȺÙ0è¨ÂÞtسî&ŠÃ bäðq ÂØöÍ°aÉŒÛö±¶M$¤Ý©' L3Zð¸ÏIˆ[GÚS-âQ€Ê6óŒ  ô ¾F}Ö»N½xô{'[pgkŽ‚䀌cÜ©#8¸.]ÜÕT'ÖÌbœ¢ò&Ìq½1‚ÉûÈéŽãgkß ÌHQ SäZwâlõ.}åàˆú›¿ù›k­GÖ‚ «2ÙœˆÐÇ® ¿Y—Éæ¬kö•ñ«0’ìÏó{‡%>¨›š¯•Ôñ•Y_Í*`d@uV¦¢uÀIß¹a¼D¦‰êEMžkC*dÅÂ; ŽT¸•n †ÑßV(O_‚òä*˜b ÅÖ#ȳ|„s5èùg•­‚Òúð,¯IâS{=Ï&ñ?5@ÂÏVV¯=²³k;E㉬O_ IDAT¼€œÒ³i0ƒEÕRZ!W ï¢É`("KÐöž¥­¡rê0 ZtË*°FÎbs¬úïµ°HÃsÒsŸ$ù²J¦ˆ8¿å¸HÛ€!ÛØ%6ÏNþ÷ÿó÷¸@{*¢–Á‘ŽDã9) OЛÎÝg'GG"mPØÑq]v¦Ðøã_Í´Hwx=HòÄ3Irn]À¨ ÙmÔzÒ QH9@älÊFÚ,Ðú NQè)}ˆ¢Ñk‡•= ƒ„DdU‘³ãkDõÇ7Q¢­²ÓÂ^[ÌÆ ßÓÉŒr0Û5Å8<"#Â$ ƈº¶ZŸ â®(u ¢áÁÇðÆ>­L´rAHÖ©˜&MäÐå`øÂ9ÕîtŒàÄ‚ó!7Ǧpõ ±x-æEz{qˆBê7ÓËà :î;Þ›Ä6›Îm ¸eñXTb£Ô¶ˆRÑŽC<Œs-*(Î<*Œ‡7$mòѬã¯"ÞÄlSuSöb˜ ƒº§jÑæh «‰ÀQ¢qFufß$„§ ¯£þ¨Gÿ‘Í28šlÁgÛ;p’å ÐU!4Ñfµ1²um„2˜dMKÀ :~Ï#ßÇ:T84†UÄøê¨@/bƒÃÞªôÛt£jXÞýƒÌÇp ìB˜#óù|^…U¿©?ïcœˆ¤ZëÄW¿L†š/rü³´ùæ™&V…3õÕÿªLF}a1«ì\š&ØrñôNƒ^ȈL—¦u&^>¨’%jðâC]v¹ Ë£W¡:Ý[#ØùêédÃÁŒŽ ?„#cÇ˵϶*åq_ýÒO˜Ø$Ñcç²Óª„iUÁ0[Àv1€­|EfZD^ÅXË¢.?Ô¤ÓŒ™&^¼é…kimŽÐÄ쿯-–¶Go„Ûâ{õýžZ_8ÑY×ý²AÎmιÙx<^Ý ¿¶C6¶±çÈüÜ_ýOuè¶GÀ¹ïS¢ÇwÄxüîãÅß çÈ®ý*sJè”7$èA9Zj~ŒÖ—CÍ‘roq jcà`XÀÉ ‡íe»‹ + y]ƒÑ´v ÏšuÄäge C¡”-jç:ª3æŽ:)6ÈmÙ—Ue‹Y£}F¡Eª*#®´«J ÈKª_ýä,Žê”¶“þ‡˜-§Ñè0Ò¶z‡_œaҨЎ«l¨ézfç3"éTÑf\¨hµuQv ¸ á!b¯AXuˆCn×N  O¼°u zÔJ„KHÅûû7š$&”AU-&Ý°gŠÜªGé:Wq¿3äÇd5 0a>·‘TÉÉ„5…¡Ìqˆ\‡dDãÿK¬•~é úJõ f©õ¦ðZ ¥/10©Æˆ¬¥>R°?Þ‚û“-˜6t%õ˜V’ ß*ZÝ.ƒh1ÛGË&sMQÊP~î{:mGpp8Û À ;¨ê$¾&ИÒv‚'e‰3¸v!àÈ'Ÿ|2ûí·;±ê«œá>­‰4SÍÓˆ > ಱ¯ÎúÂhúê¿ïóU HúYÚÎOò,› Íì¡×‘Ê5#Z—ø„˜Na«Ë¨g×`y|ìbij@sNPNG€³QË&žä€^…ÑÀòôíÌKo¸´ìg1iÖÕé’xÄ3Hcí+•f>ÚyÏœ°EVƒë®w6‹2k(‡ÈéïPœ·ö$dzž¦Èóšäz=>”€ð$€$£ÚÂÎ|Ù0I–˜Ýv“úA^ML'Þ±¬°| §‹+‡Wé‡`ãôk–D|&çG áJÎjaP8”GC ê¹(#ŽSý Ußî´-º¨MJ]³åÀø•ÕÌ€órwsÏ( ZyJ}‚|¢^\ ¸«àº7±`2Ýgºÿ6 ©—9”¿”C/ÿCvĹ©6ÝŠxɨÛ\ºm«ÓH;Ç%]/üÂ’¯üåè9˜Ž@tŠ ¬ägª²žŒ'ðp²G>Œ¥=# èQb&ª¯Bô¾¸Ngè7ÒU€.4Ãqä÷ÈkÁ‚üÕU@Ã\Ôå~¶}öé§ÿ×· ÖGÔ·ËùË®nÀ‹±¾0HÚyU˜ÔªðšUa"}×_õ÷*6Jc!P6BwÕ @2ôa4‰G“Zs˜[ïÁòøԧנ.³6ýWª# ¢´`ÙÈã¡xé&Ln~§^žÜs³‡ÀéƒÏ±|xzfMúù:Јۮ(Àdó e%ƒŸ4œƒÊÚ&ä&7% Lã,‡I‘à ËÃÂÃÅ­¡}¼¥O_è‘xñ%S¾ÿ¦6®ò¸U3äY#}¶DĬµÎsôÞ{ï=oƒç†²±} ÞoÒ½Ëh^%ŠY¯;íééÅ¡ˆÂ T Ú•oÃPô xÆ×ô~_;w€ÂÔ@HîßÃâ0‹ög…®ÙȘç90®ª†M2Y–P„õN´¬wq$k0D±^t}è_z(Í‚IõSÒ²3A·F¹tÎU¸ ø^ÚÖºf.Z·aäÀ"—I]ŽÂJ˜<¬ µŒÑš3T¿’8„³ °¢Zaí— r§²°è29~§úE ­Q=8õÜ. cˆ·ß¢nïY1ÖMbaÒÞÐ s°¸9Üö:žÌ”ð”VÖÖ¤A”Å^Øz Ë“Way|ìrä•–¢]¡½µÖ ×oéLqÅŒ®ïÔýWÍäÅ}(?3‹'gîá±e k€¦§e—ø´½`­a<™dh°õÌFص®aV•pTš&Üf;ˆ¸Ô“¶ˆ…Ê:¨ø¢E ìpµ°req jRÓ¶NcdÕñO#Ôº±¦í<8røU…Cþ—ˆllcûÛ|î·¼á±çÉwÑÌ%&Y…ó©8ÍÀs;ÍÅqø3ÔŽ1¨ï£ìä>2Ÿ%à†ÁðÐÀC $eR×KuSôý|É–ÆÀ²(àć½V5ì-–°½XBVÛÌÐ΋„-@Wo€œyÒ1P@F:—¦á5ú>âkõʱÆÛCŒ (`„öæ#o/ÜÇÆ;Wš- k(Ù²Rm3I¸LÈ%¿1z~)—µqº^L7Ó”ÿÉí¬0 ZIXL²a…4l‡€:P‘ Ƕàl×Iù[P†ô5škú·fäG®Vð‰û}è¿nÖ8·ðJ+ºê&Î å³Ú©1 jK­¨é2'íž.ÃÔ ™ez×ɪe»ëº›djAKíHL½È+u§6@] SE€Œ~gQRý²N ?0‚ÍÚl4÷Æ“æ}­ÐðñÍCg ÷³}ºBÈóÒæªÄ;rN‹®y÷;ȉ„{™ KÇ;&5é²;UẎu ‘Nk‡0?¼ Ó‡W­]˜;pIìÂÀ/懈VwгUϲ ¸qùL }jŒ¾¬3éÏuaNëØg#ЬbMdóìhp¯þßë÷²à›`ì‹yl\Aß.¡…©¨Å4ÝM(Þën€­ îº8L'lšìR¤,­WsE³eG»3Ü~¡Þ¾ýýÚ¾qÏÎ?6³Ïî˜úp¾NË%µ¾ºñ?³v¢ÍyשŒ$»bžIâÿ ä¸\6 ÉnQÀVQ@žeÍsxÁÕ†-Â5¢w¢#ç¥Uv­h:ùéäY€ÊÆÎe®ªªŽnÔ%¶ÿ#ü{#$ÿ.0H6Ì‘mìf£ÑÈ Ö4üÂæáHChŽ"·×õÍ·94Û#ö}ØH‡CZÎØQê^ ¹fE»ØI£ß“ù-šçÂå½xëÉÀÀižÁd4€kžIâA’ÀÒ”óÚ:AuíNYµP*£EôPɳö€QLš~6@@éL0 {×|ÍØÐÌ]¢ã4ààœ‚gè™!*GË èZmy‚Øg=•#³Ð0HÒZ’uŠbR‘M®.ª'a;ªÌ òYÑ!Y9¶ýÅ߯f–‹U¡@ õ ÌÁ5ÀÀͲvè^±m(Mц±C9þ-ã‚þ)Õ]³®T ãw,îwºn0øÿÀï—Ö ‰A2ùN'  …@@Õ[Q·¹€$mw•ö3ˆüÞ;ijfbQJ`:VóZ¼•y·wàóѾCÑ&)ª#Ô¨ø¹\üNtÇ*?jʘ#"Qe5Fò{éªE‡Za©ró3CO'ü¶™yéí„óm9„Ù“›0¸ Õk0æ\»päÝwßuÿôOÿ4GltôùEÁÕ‹±UYcΛu& ³!ÓûºŒ,ë2ݤ,–5!'._šƒü1Ù}û[»•ß.wì÷aˆ¯9tcÚ9iÇg=ë ç­“_ºW!ƒïV[œ[ž/^¸9ujBQE5(Å„ ³~(³`Š<]3ڳý·ÝÖíûîôÑûfþ駰|<…@‚ISQ}†­§F“HH¶0#CŠâpˆŸ<ÚL7%Œ–\ aç ÓD¨Ã†ÓƒÓ±Ú2¡TnMªž°£YbóeÂo¾ fŒY"^ñ§´àoÃ?Ø€%ÛØ7Ïvvv\mÝðóÝÈÙ"_@37x³¤i ‚ŽwïÔ@,×Ì•zŽÓieõ·]C¼¢2ݤ)r)ãguQ\ í²6)€§Û9ŒÆÃ$ÙZ, ¯$ÜÆ©Ýd~º€!IIV ~š¨jâPd(˜wëé{Ú,’Ì, 9D¬tžfÁQÚd |]—8Û¨ïIàJëx“ÃÎŽºZÓq(A.ðòÒap¦- ‚¸ßÍïFû¬Vœ~P!©~1V„-Ôž_[”¶×©dA­-Ã3R¶Y;òî^ŒPÙ¬Sô*¥×XŽíT“ÌupÝR…Å‚²0œª"RÃúêæ!^Ú(Ô<¦Œ‘¸»Ed' g¢>‘„Î0˜ -ó‡Ö¯ŠÑœcÁÔ.“›’æs•@×¢ló…k–ywváîh ¥1„í0sÏl’‰®Œ =ìÜWÚ‰™_ÄQbÇÜâ¡ñ;Ò¤÷U™r\+À‹N*lÌ‹¹ZÑ\äà ŠÃ>‚­†0;¼³GWÀ–¾ìÔ–p—Ä.ñyèÿú¯ÿzéœë88_¸8ï¹`ä«·U™h`…ÆÈyœóušëÂsÒ²œ7”$ùÎg–æ>ͦænµånÚ-|§Û7›ÅÃß.šMüؼ ƽ ™{ Ðtv !¨$¥ÿ‰èÆ‹; cd¼ŽPŒ^µ{¯¾d·n>2óƒß¸Óåþ©ÆÇWiµ¤ï 6¹ìlŽî¤ÞA=Þ¡ñÿŸÕ5¸å®ùÀ¬Ý­€ Fø6–vA* ®NË›R•SKéÏëlŒ¬7k­ÉzÁ‘Ôž7°äìλ±}ËíæÍ›îÐ̧9sZä£Ól§ äЖȉÖLÎ^ ¢Ï ™³é˜8ä%“$‹6à94b¨&â̾ÀdÞ¤2A:ßÅÏ`á4Ëàt{ ãñ®.–°3CQ[щ¨®ÔHäÔ&´w"6Œ‰ë€×ºNÃ:B¹¬á;aLD»ñ:¬9iOíè‰óg™ã‹Ó}‘€‹xÓG—%hz˜:^˜ïÐ5´Üì/é¥EJ£é©i3ã¨ÎÀpN]‡²ä(6 íµLÐ<ñeÍŒôf„:®û…˜2Üm‚VŠ/×õmX"ƒF)þÖ-SÆ„jéš•"gwuhHq…@=—ö3ÄèÒgã¿4Ûƒ®m¹òDX”ê”ÚR‘Ÿ£wMÞÅä6á²¢g"Æu7„v]Cø|w GMv>…þGà‘YOÜ÷ÃÖJ?lžEG²ŽmK~†“ÒÜSu¡f5iÀI=› m |¦Œ°º¨mM`4±?àûi=„éþ ˜?ÚƒÚ#íµO몼4¡ÛV3 –UUuTh¿ p±=¾~[Åæ Ó€G_Zæ4Ԧݱ.µsúý*KÁ“óf°ñ†«âïºc÷(ß1¿©¶Ý÷ë¡{ Ž]<6yÍr÷'`Üíîë&½Ë¢¡íÑ<—ÕY´,aÀ¡Ë Þ®óÛ7ìèê;$1G¿ú1³è<;ÔÈÈt¯âxÏÀ áOø[Ç2ëjr(4ÀhSÆ%Ïæ¡Žt+ò9èÕøxV!â¥Pƶa–llcϹyvò?<ù‡S0y+ü¯²£°KmÔ|Àó‹LNš ‘:CLïTŽ+;ÈɦFäôìÌßÐëd3ƒY*üô|§AåÁð|‡.º'3c1Ï ÜŒàpPÀÕÅ¢o5••jæoRYRÆDß˃…uN*Ó˜29@ûy±“LN\§=u  v¯Õ¶¬¥º¢eÉp¼0 a2VeÄæh¯¯2Î8z˜6NŇ)~+„gãîTXŒ^ï5¾Nœ}€ œx kI¯³ÂG™ !3á÷–Ù¤˜# äµm”@Æ: ’6µ!<†@$ 2ê¥vl`}X´PöyþˆH™4 Þ/¾’•­6ºgJ }ÍÈÆ]ó½eІê[ƒ/O÷4 ääÔ¹ôÊ(ÁdG¡S´Nå°N÷}êF’QÆyáÕÁîîìÂã|Ð.hµI²n5&aQVò=)p N@°z(Ð6¢¯ÉJ`÷GÆjwmÀÕ‰ºŽf½¨÷Õ» AKÉVL÷oÃìÁn³°'ñcŸ')wx—Ä.ñ¬ü[·nÍþÕ¿úWs/Ú¸±o†¥¢«}šÚÖ¥nN¯±Š]’žÓwUÖ®œ•ýF_Ëìïf'ð Úª]oáw«¼9ì€p·!‡ïØm ;½˜I)»Ú"³t÷)ZSÄ»MÑàÈÛ0zÑ™ËòÛe1) 3þ$«g‹óŠÓ†0-¹RßU~qœâL¾QûYÉ›ɊiÂ.ÂÕÄÛ½YéeGè#µ˜êÛé#KA’})›N&“µ¡MßÛ€%ÛØsf>‹Ö7oüÿý¿>n½¹P~픓6F èk0D;õ æúNƒÀs ‹6ôÎ5­èjŠ`´å Ê~8Áý׎ÖN•O9”i¹"$Aísx=’Y6†É`Àá6¦¶ê9úrQ@:E`Žfë$óº‹VqÆŒNx:Þig>ÚEëVœE•Ck"@w‘A»ïÑÇ¡¢RaÔ’³8Úu”„´´‡´µ¬{\T&+nª0CÂf#uPÕN2YCŽáû†ÿ[^3=š¬: nJ>ƒ­&Iº|wI߀®Àú•þMÏ.ág1ûªá«õBâïœjàXTTÊÆ9}¡¶á±Z\ÔG­ŠPá«°…4a¨N"ð€ËÁûƒò\áwk LC¸€‘èq0^'S½Ø$u î2ó[Ô¢9ʈßÐ\2*„gnX~Ùy…v=®Biø:Ü­©Ok°Q×O"ÀªÀc &‰z«rÓG·aþhl½9¾þÃNy)ì˜#£ÑȇÕô ú­ Ùè…\>[Bs»ƒì¼a«ÂlRô\svIßñ}@I/`à³æd÷̉}O̯«Ýí?¶£·ßÉàåÂÀ(™˜Õ¢E…£èÅ@¬ÆEÁ»ÏÎ6ÀÈÙfŒ9¸yóæ·9r–mÀ’mìÚÀæÎÕÑd>+¢!>œš;Õ¼Ûçèó.´ÞŽØ1­]ï„ÓuÉIqÒÉIê'tH—­šQzç[•wïe£"QT:Ný_AyH\Áö¢„íeÙh’ïð;Y‹h0 žîu =¥§å—eDWg…ë’7`Dü]ß,}Ù¸¡#LÚ—‚„1 yê†Gi&€œ§ËÑÏlhÙ©Þ!ºì¬Bß©µ‘ÿ¤Rys •À¦º10ÉDÃÀHÂ~ò)y­V¤_k`D@)#ýJ¥Æ¥º!-Œp[¿{q]Èýd ú%T!: ?ÑÀAÔ‡ø6ø-pþ"!TBE4`DÀGÄÄ&0DŒüNGkGÚÜT¥3Ë"‡G[;­ðjg®™Uº_¤ƒ?¯I&ä¹TvTchŠ˜"Ê£Ú¨~ÊhI†Þ!ªg+@àJó.Ñ54:ÜÔKË“]˜>¸ å åcq¬w¢zÅçƒãÙ·ÙÚÚªêº>öÙ@\ºKÚ KäòZŸÈéªì2°BC$ýNÿ½Ž5² 9¯ƒßwî³²ÚšáéþãÛÙlaŠñ£˜bxŒtГU²ÈÑ;H0E £‚ ¿õþˆCê²lÀ–G[+5¼Nè6jçvŽÏõ Ïh³^쥻Z};i wèç×3Q±ÐØîÿ®¢àOi: gc_È">ùéOº‰‰ìÚ,ÙØÆ.UU5wïqTEÄú vb”2&¦„k`$wNõÎz†(¡W1ÉÔAs¦¾±cçd6Ô`ƒšµN ú©ö&"Ö†Æôî9æfÄ Ñ»ôMv› NóŽFتj˜Ì0®ê& 0Õ£ãšÕúcRw?q¬•’èªÈn·yó†Q ©§Vrl¼ãâx1Aõ%^t ’ D[ú¨ŸP¯i„Ù#‡¦WÝ0 PYäåïÉ IDATk@Þ|§P/ê£JF˜ ¦íKV‡ÕD÷o¬C©m§k*ÎÐ:TƒŸ)¨w+…k‘ÄWãã@õ}Št_û½Ž¼àòi0ðCLž¶­€_Úé +P_Ó)Ђ˜Ê,ðŸKj? ñÿ\hSÞþŒÀ œL&ð`<ƒb $X¤ŸFaZ®7M€‚Ù‡`pÙã’IYÈ7¼=Vn䜋= 0)`Õ0@¢Þ7Zò³î *À„2÷Ð;ƒPWcX]ƒÙã=¨¹z$­ËŠèðÓŸþÝOJÀ÷à2Ø…#ׯ_¯ñÚ!©×3Û#—ÓúŠ¾l4Ðã\§Ö'²ª¯³êºç¹ö=ö˘AçêrõI åì²|ùdŠñ6äƒ1˜<‹cIʼhšÓkF|5`MÉr=Z(¹låÙ"eÈQÍ@U[#sŽ0¤HgÅ9££x.*£çЂˆB)-¤äˆDîô.~fç°®]Àù“¢óÚùâ朳u]_ñKn°dcûl4yŸä±u®òZ[¤g†Bâ²³»ÄŽ¸cí œòfP…QЦ9ÍÅ}Ϭ;•³J[$9Ô4\ÔE3"Dn±~š(‰µQÔC2CC;aì+'K€€©×$É3(Š¶Êªa’LÊ òªŽ*9C ïús9¢ò»äoò¨úŒ4Vˆ£"ÄÏ#×P#¯N`zç¼³“,„¨}PÝìé%YT0î.Âu³šDQª&¹õu–VGB¦’]DÎPá+­‹#K=à  ˆÿšd½!l‡Ø–ÑŠÐZDFÝ·4 :°(¢úW짚F£Œ1hfL¨8ÝÒõbœù…C±¸.® AêN¿›.©WéSq?Ý—`™°?žÀ£É¤ÉÅϺ{Òú=Ô‰ªMýŽñØ`u=Jì u¤Ãê`0,+É.ñ7ø%Qòx"m,å(Û0{|–[`ë¬Ù¨•ïhÖÞÏïä~Q«¾f»0pä­·Þ²óù܃#Ý÷Ë‚!f”åùÕ|«>²Óú´ÜP¾Ÿ¡­K…».$fÝg}€ÅÓ°…Îà YWŽ/bçYš».¡òÿ–SXž@>œÀ`{ŠÑdy.K ”¹=YÇ¥j0ÇM Èwô‘g‰8[‚óérU}Òxžyh—rÛŸ#”‰Ï÷ôdgÞ)Ó3ÉžŸØI¸šÖêÐÏΡV ø¤^ SÓš-z…Ný9 ÷ìÓálmÓb•1ñ{¡k'Þeä1ÄèÇýŠÞI”ð3&[Kû8Í6‚ä6N®Íº t¾OškPû)¢„ÿɘìª>dt7a4³G7`y2ÿ†/üGuº4îÂ%² G|ªµüÇœgYö¥EýÌÐdÅ £ëÅíÑw³ÁKvV?\Þ›þ¦ü|þÐÎëo£hà35…&µU!2g1V…å@›$½n_Î]•ÑæiA“3§\ãzåãwË,«%”³ã$n_b²&Ï#zBçÓ´º„ÖÉ9@Ó—ì«ÐU²UF Ñ¿™€Ñ™h±^$—Û̺–Ðɼ<)MXó›7â…§Ìz…Vƒ~Ì$dBŸsfa·ò<@ÇF`õÙ"–eYοIÏô5Ú,ÙØƾ»yó¦­œ=ÎÁ,ÒZ­ƒÑ˜x9ȱ“ÞZ*.™¸M EÞEçtw!9[Yºi 1«éÉä¡ÅEôœìzÀžn=¥á/zW¼Ì*SÀ4Ïá`XÀÞ¢„ÝE YUu®‘†ÕHøˆl¤PžPuÊ©ˆ 8áç¢L'’»9µÚ‰õ»øÒ®{¼nͬQLê¡ÃR\ÐqÑfÚ,}~ú`H$•³ÐÐñÂ>€A*Ì$FL»z'!–#ÞÙo’&J“pÉÄ)tùÞN@!þ§´5„ײKPîGë·ð¢É¶±¸8श£— b4'_rqÆaAa]l¥l ÞUPN¹`úÍS⫺ßDZ4Ýzµ™£Ñ¸]Íó6Œ¦É,c¤sª>¦ßçæº,”¢Ç¥£Baâµ­{ºCŠB6æ@Wzž” 1•G{Q4¢~'kÚÄf°8¹Óû7 š ¸£ ^9¤N@ ŸÆ×äæÈKnà%a\8ò“ŸüÄýøÇ?^Zk—_æ:ùíáöðåñ[ƒkã?Æa¾ Y>,næ»ÅëÕ “wNS}6lë¯H½ñnyž?u˜L_ Þ¾cú„@×ç\V^VË*[•%'5ñÀQço¤ÊCgH_29ÄÔ; ;ó’ŽaÔ[¬®'@Œ¯a}p1ûûÆ|`ŠIÍçpžTà`äÙšGƒÁ—·7¶ÒÎK6¶±Ã~ô£Ùÿ³úç)”å)ÞŒ¼Z§29¤`G $s§ã}Ybv¤z#ñ.zœ ÕOÃo”G9N~œžû¯I×Ñ@H¬³tTˆø«z~Ôk×IôœME÷qKu^À,Ëáp8€í²„½ÙŠºÎŽbÈêÐõÌäÏÀm 6K@‹ÚGl µ¦à¶ÓDqBY#M­GøÞjGJ€ÚPrœùôFPbIèE¬¯Öj6ÄÇúû™Pß6¤ù¥P£ú“´%&mÛÐñ¡C©¶i³Ó´7³ ’°3Êý„É~ÓMÔÚJL•ê‡®á—„ʼnf¡V'T Ë1"q¿/黩³«P(1l¼ fC3; 7¢#å½…ˆõ¢•P¤¯Jûqfý^@2> ›gp8Þ‚;“-8 `g†‚ž =§ §+”Šú˜Á§Ð·A–GÕÑ8<»…®óÄ fÈÒ -Ã^&\ÔHÛÚFxõ*œÞ¿Õ´}Ä„tÃúñÐ#àâ´vËÓÞÂ}MvaàˆGƒ^yå•åŸÿùŸ¡E¶æÙð»“׋¶~˜³Û Ò ˆY¶S\Éþ,ßÎß\¾8~ñÉÉo«Ï›8ùsšfv¬ sY'¬šëÀ’ôÜ>­‹u‚¯}º%ÏZ‹ä, …­Å$¬Mòûk‘Õ&žÅº )S˜>„Áöµ6äf<‰*4P' 5¤ch“‰f¹hEWÃ7Zü“I¼IU²DŒèúF ïz85ÙbŸ+ñnYÄk•g#– –1 N%öÀÈÂàg÷róO²ì¾Ç =pW–e,—Ë rq6].—›°Å‹±,ÙØÆ6v{ï½÷ì«òæÉŸüÕ¿˜´› $ Ä©Cô®-Í»üU:ÕŒ "ì)ô„í«ö0FÔ=hË”6RxÛv)‹C›S÷ê°1õ={!p /å™A6}ºÚ!mºÒSÌaVp8ÂNYÁÞlƒ²’Œ£ ê8Éšýì“•¬Ê™Œ˜†wúùrÊ¡dÑûèrcÆJÄrÕi^#DþÏ»ìá)4FE¢ÄBë"ZrYJóKÙYkEÖ{!›MH›ë·A{Ä$a>6ô) Û„wEvü]T $J#£€ÿ‰ß›jÓØ#áJͣΣ6Ðbô"¥€S ¢þ‚P¶ø ¯¥¦)[ ANœoê .¥ÿ7ωòîÊõÕ;ä\“¦÷áö.|>ž@©€Îè=¢2ó™ØKzS•À}ê.IÑ °p½% \‹–ÉûCcWŸî ÷ñÌ´cÎâ¤|an)@³¦÷k‹£0}p ꙑ~ÞÑ4, îgÿtf/;ùÂÀoÃáЧé™>Í9f`²âåÉõÁ+ãwó­á[˜á8EÁÔànÌ ¿2º‘ÿËÁîð{åë‹_/>9ý]yg¾‰—_a:„¦/Ìå<¬‘UÇ®Ê>“žß¬ÀÎ{Zæ³lUæ›óž·ÖôÎ Ä¿wäÔ¬m^‹£‡°<Ù‡b²£½kMèÉŒŠÄðЋ%ÏF©—‹„R‹ ˜¢4?Ð……Šé¯Ö1pêÜL—ÃáùbùvnK!i¥’!|L:t˲g-ú¦])”SÌ>øCžýü$ÃcúΗk86e  PB`Éƾ:«ëº‡µìmlc—ÚêS8íÈ`ëaòn­Ê¢Ãm´£éM¨ÝÚFË@'iS}!+1›ƒJÌwzµß‡Ðö‚ÚéT‚ÜW(ÇëòÑóëŸhÑ@…AÃ^Ÿ¾¦¯Ý¹10à ÈáZYÂÎtò"WUÊh’DŽ¡ö ¸ág°­ƒÂa!5v$“Ì~ú(ºF½Ç‚ÑwIˆkkÈšFƒ Ö&@J’£Â†<¸Á¡7!L&Bt‚PjHóë ezªA¡Zh@Á„²8Jù›k-3ƒ2ÎÄýµqÐM\6„ÜPšWŸXX7•%‡ø­¢6# ª:k5PB*Y”§û[U^ m¤:Q}…ÚRõf^‡ ´ZqŠP0ÈÚš±3G`FÙ†D\X@Ÿ²(àÞÞ<¡bW• Šú’¯ï $cŒ—øY©ï¢ªæš±â^xV2P¿¿.ÞÆ'Ò¡÷Z/Ímè…\‡(ÅÕŒ+ïM¨>®€ÙáM8½{ lÎñlzÏQѸ–Dy®y§ÕÎpÛ\*IŒ GªªªŒ1>´†Óù®³l§on½5~aòC˜k­ö g2î)¨;˜ä׆׳ÿ*ßý zeúËùÇÓßU÷'ù¼—ÝÖ銬K—»Ž1¢?_wlzÍ>§=½Öºr®û¬ïúëìiRËîHâçkÀ$™¨èkg« )§‡ H2Ü»ÅhÂm@M`Ž!x[.›4½$º*ƒŽUÕ‹4f²Øv:‹ëcCåíüäáÎöß'õ¯¶g³ï–åÛhÝ®Þé’‰Ÿ*‚îÞÓh§B]\UX˜Ä|!§Gþó<ûÕ —f§½¢(ɆIòÕ™1æôøøx£å´±mìR›ÃbáÀž¶¡ø!¹©fb€L§‘X%}fÔoÈ|§W®²M »¡¹[ÓDâMÖ£›¾ÍPàs’YF;äzã@¯Š…M³E$d(C™Eñs±˜#;•Refྒྷ¢€ëË%ìÌP”U÷z´6éaãðŽ¾®5å Ñº‡ÁÅDФö —OX!ÝöS"­qN•QÂ^t *AÖ–ÈaBË™–Ýa‡¬Ð•4½’Ù–&ô¥î­Æˆ€$?Måpt/ X‹ƒúµÝÜ8Þâ –¼°-8£ƒœ‡Òç[k´M˜q Bf¨BšÞñ+{v1{IBdâ°*é5t9aà`ÄîQëTrÚU»àÓy'`>ÂÝí]x0´àõãt þ¶5‰1Çt Õ„º“û%`*ƒÂ,qÄ.±13F‡âÐ-Y÷¥.õõQ:Qœ¦!42zñÕÓ'/ÃìÞ.¸J§sFÖÌ¡0ô\Û9Û~a©ÇïÇÇËË¢7 ŽÔu½¨ªjv¦Ã™ÌnæWFßÙûÓÁîð»`pÄÈ-MÑ-`ž„²Ü\Íníü×Å•É;‹ûÓ_.ÿpúQõd9ûJð9²UÀHÊ Ñz"©¥ Ž³@‰ó¨®ºÇª2ž¢¬Ó"9«Ìç3”m µ%1u-ê«‚Hˆ,„È4 ÉìŠñ. w®@>ž@f2?êªr٤굩XòLc»Æg{Yõ¶%¢=5æñlgç?Ž—Ë÷'‹Å[Ãåòûƃ$ÍÐdmtîø•÷¿ZéС&œ[ñxßdÿïÜüz¸8«ú©ïùpÍ$ñåß%ÏάµG~øá™í±±mlc_§e¦>OØ)H¦íXƒfŽèùR‡L(gKœ``ç¢wå©Å˃cÜ àu€¤ ßÆ ˆ.‰Ú}áOuy5¸ƒzÍrõU,“>gÉuîWhÄ0 ç,»£y+Ü:_Bî5I\ÌtÑ<­E œ˜Š‘Ãm8¥zmê§7Œtñ;'hð$f©¯•³,L©/ƒIíml{eGRì –˜à6Tl ™kâPl¶&pŽiY"ž ‚a$[P+ÞJ"°’¦%z`ä ‹ÿ^ÀŒ®ÍkPˆŽ£ðªsMv°Q÷¢>áÀCžÛ†ä‹ä웸$N §€¢Ûù˜˜qMxƒ$K@UWµ#áó­8ðÉRÖ“ùâ0Ójƒ$ý¬mÓqj)³‡ØÃ%‹4Pƒ ZRvƒ1ð¡ÊËa9FtaР*£1´^÷C\UN`öøXìûT½²‘å¢a?¢š8³QèîþÇÿí½Tä… G|zçÜ̵³G–~9šl§˜¯O^Þ˜ü‘š[0“¸Àp\“®ID¨¨•1y!ùoßÖEvküòοÜ}gñùé/—ŸžÞqS·ü¶ ·’3©Më‚ôi¤ÇêkÅÄ8O8Kz¾{@ßwî:˪ïÎ[®GE£-fhpA‡z4ëËNâ}Ö™ÅÉ>,Ož@>Ùm„[“톂æ…W­­:$)Ëéábº%ø5ª¿«B™úêEggƒÁãiQìuýÁÎbùÖhY~'³ö*:›û´¿íó ÕÏéI©}j ÷ï±Aÿd‡2üO÷Šì·%ÀSÇÉø>íÿy&‰n’x6 ×ñƾy–Ÿµöd4mÂj6¶±]js ·ðiDZjøÞÝ©Q‰t§ÚvEµ>†Äàcg;;Ò´`ÇH!.vüMD›Á†(ÄÇ‚rúµS£@ Τ‚¬ Hìˆ)ê?8~FMj‰B aÑDPFR§šRŸÙfšepëÞb [M à*ÈSH}v-Ò !#Í$¡K°cÖo$cM¢£rãz²Û!Ý\«3}M:´YBñ¢Ç%@J²BaúJƒ Æâ¡H"Nzû¸ºóHߴܯ4G×q lQŽ/-· …øpÕ:Öiç›@Žðl!] $ÂU-‹ÃÞÞÃrD€nO¤àVØß)GÆHª3·Hx–©4à 49<ÞÞÏFã&M¯fÁ¸ÄŒtuÂ'ì¨þÖ°Äm:† !AÏ´}­ÒPŠÓëêçHÂüŒQlÑï<½o¶vê:T ÒF"Ñ_Ã@µÜ†Ó‡·aùd$c) OƒSºIôYÓY–«4n<1ɺÇEqú­«YfY6K"ϱ ^Ý.^Þú^1Éßôp†±“4ˆëpŽ*¦÷ð›n ÏÆÅk“7ön7ÇŸ”÷§¿-ï.Øy5³ û.ÐÀHŸ3œ#g1BÎDM¿?ϵVé–œ7üçY„Õ¤ç¬7‰³äÁP ?i¥x™L“P5I‡¡<=€jv ËÑŒ÷nÄ|Qž,ôbGbscª.¨nžþŒì@i½÷ÕyƒãäùáA–ý" ?˜TåkƒÅâµÂÚÆÁX[82ÙÊä]šæý8[Îî<,²_<Ȳ»6‰®|Z£þeÙ†Mò ÌÓ³,{òÊ+¯Ô~øásÿ<ÛØƾ¹Vì§ÎáAºßRµûJsh è /úo €R>˜~‹²{Äßfw$³2ø¯D,V͘g1aƒ°ó¤RóZš}ÃÄYc‡ØuOzýÓaÔøty³l Åh»Ë%l/JU5r ÙÒ( „™ãü0Ž uJÄ<уªY".rž©ðÑþ“ c ( 2¥aTéý¦•¬ñ⟤JP•Êk¸H,V…PóÚl0 …º²ª;:^69¬FO¼³Œ¢B"»`'Û Ã_#B¶Â¬ L…àÃÑÑEx˜ ;Aoz{Uß½ çt«È;huÇŸ#BÄziX5€0 áÁÖ6<öˆÔ›ÖÎ á' ®ª°=V¨þÖ„WQ›%Lp eu®´§Hv¤ƒ„ÑøAa>šY¥ÅVYƒ$yw@×­uª/"X›C9݃éð<¿Z…ÔsacÁ`ª$ÊßD€ gœýäÖ­[ˇÂe± G®_¿^[kgY–ùŸ™EÌv66Wò×·ÞÈ÷oš,Ûé„b…¢2ù¨3X<ðô h T´ ‹áwó­Á+ÃË;˳O÷gŸ»ýêÀÚo“„œ^Íé /9K"=·ï¸/ÂÐÐ×Jï– l_™ÎºÏ³5¥+Ä8Fxõ?FI¤Þ¡üÔw-Øjú¾‰v%Â#Y°%ËlÂj\ÝÇ9¯nKÚVÆWÙéa‘ý:/lµ;X,_+êúÅA]_A”8—Ñd'ضJG™ùè¡1ÿ|eϪY2t¸;ÎÆE£Å2;=ž—˲t$©ªœÞ%ç3D´eYžú4™?ûÙÏž‡"olcû–ÚÜÌKgêCY  v„‘w¦5›S€ N¤»ªzÃ"†I'ôD™v6´cÔ7ýè“n8þ<„-@Ù%ÇXò’èˆ%šv35äŽ h– —Bý‚ª^ êa±FË„&Üæ¡·`«¬`g±„IYAV[•¾T³W@7™t"¸„½þ¨P}îœlRÑRJê'g”Sî’ôÇ ÊÄÎ<¯óÔQ²áû+”(k¬¹ò=íÓ‚)Û€®Im¼›ãTý¸¨¦r‹0kig>ÔK×ïWzÚAgCyíÂÐ|L@'†/é¢ï ÞXNí)]»šÐ ¥÷ÚQÑÞ¹rØ…ÑÚ‹wÉ*Y™.sò¥f%Ðu5»…@  °ŸíMãpž9l•5l/K˜,K(êZBn’ %g‘¿Õ¿‡çp˜®¯’cô'*…nô {èñOιábÀCNOt20¾/û³Jß"ÂÄ•ëÕ^Fƒa”%5S2¾pˆtÕ²1êw:‚H³J€ÓàºÄ§ @J>ƒ²9-@ˆ‹ºk%uw°ØývÊ÷SðI—'öåÚ)‰¼Y Pæ9ì'ðp4†Ó,ﶨ1T{th^Ü QEE`$ÃqL–fØ(ZœϨе0‰x©Oªr¤ûà [mÛ ¡.Ç0Û¿óý ¼ª”C:½WƒÊ*æâhæyU>z÷ÝwÝOúS¸,v¡àÈßýÝßÙ¿ü/ÿrþÚŸ¾¹;xiòƒ«£7`€WÐëŠ@Ï$!¬¤hða@»CQKQÂÞ¡Ñgÿh$;„¹…eé~¾M;°«Â_Î+Fš:Ê)»ä¼‚¦ëRÆ®Óé³ÔA¿4 -64*ëâïäwµðR#Êéäªàt:—CÇÔÕ¥¾Ò`é¬C*=7Àt–(ojÖ˜ziÌá"ËŽfƒÁ¢¬®Œëê¥á|ùZæÜ­`~b²îæ7§G5â3 g»2‚ñKWÝ7¯¸w&¼€óñ^º¶ƒo,Ü£ƒüäñIñÙÝ£ü°( çSû,Ù¤î7Ÿ]̇B^ƲmlcÛ˜¶ŸÿüçÕŸÿ·ÿÙþËï¾>‡d×6õ£¹9ÔZ^´6zŒ…YÕœ9:•/]#ç8¶XïC3A’èdEœ &ú&qâA‡«¤ß©µƒ„©úb@HîÕ†[Šœb¯F¡A ÁÎ8Be‡N‹ †ÃvË ¶çKø9¹³+-Â\f!*ÆG¸5íV÷#Sé½”Gß°ågT!ý¨CúÕƒR½4Ž6Þ IDAT°Á `qîùHÅ2 Ö—cäy1é½@–QNwd1XÑ…ñÒ4ZÀÁ´ëá(‹H=¯0§V­©Eëãèˆh%Ü´§Ãª4Èèˆ-ÅÇAÏ»ÕFƒ/ónmÁ£ñ–Ø.›ÛЗP±‚lwüPmŸ¼i¸ÏJTë 8Èq¬áÇ7Æžs€JfŽi°–ËMý¡ZlÃìÑ XLÀU2ö ê»ÜŠ ðÑ6Wüî¥Ô*tîÔàèð²mÀ](8âã×ÿèßÿɨxcügƒÝÑŸbfŠ8þ3F0ôøÓ;·õTˆŠ)#4´EjíƒYP?×YŒJsš:µ}kë i9C{¢4Y÷¹.gßï}÷Y‚œõ÷EYÈ”ŒÎ+Ñ®d¸NÈÒE˜ “2€Ó® DBo4˜ê â…ŠJm¸ª]¾hõµ—¿U8¯‡ƒ{¥+á`ð»í²¾5ϳò 3÷ÆÌŸUódˆxkÛí¼vÝýñ•]|Ç+!:Ó"ïe9îî渽3Æ—n﹃—¯Ãç²ïÏÊʇø5LJ6l1çÜܘ˕~cÛØÆúÌÏ;oÿå¿Ø·oÁ)€±Æ¯4Æ!üD”{훨][r"1M,Q¬€„F¯=m’Õ.2's¹^-hý<—x¼‘îI´kOëeqùxÒ‰¨­ÉFb@"g]……ƒ•Axʶb(¼EgÒáÇtÜ6±>{æ " Y(¶ab©þ’2hTæë Šÿ–sºxôoºÏ·!(ÆèµrT{-{†_`-Ò+`#ªóZP"<"̇¸»åÃhPÁTn++a'¬S”Z8÷ÍD3E§7VÇ€“þJåYTÙɦj4x)câFœuK—Å™ÚÙµÈZ¹Ø郛PÀ«2¸„*ûNú¢;ò‹ ½VY¶[¦öà“#ãž™?ð¬ìBÁoÞ™žL¥«óñÖ²&ŽK¡°ðJú]Ÿ¦Hø&y b$Sæ±v°¨ëªEòô ì\aF¦xž÷û2Ñô –¦ß¯8ÎP…&GzßôZçY}í’§ zÖ2/5¦)uüY¢â®C^:çñßr^™°‰ãô÷®ÑWBk|m’ÝËrVZå>Ý—§aéïkÄ ×ü¸É7ÿ%EWµ ˜—¯Ø[¯Ý¬ÿb2Î_AgYÈYï4kãFã1ÜðÆõm÷½—¯UŸÎð“Ïö‹Ïš©?’\7 IkιéÑÑÑ&SÍÆ6¶±çÂÜtr2½{ë$ŸÌ\±{ˆÙøTíp‹Ð$(½‡Hk¤19>³Q9%šU¯'càBHT G,]S}À;ìœ=$ ‰„ßAœÐë‹ ‰B83JoZ¹OÜsÖìžk±dD×k¯ï/gEóÌÀÁp»e »ËÆ‹ ‹Tª²a ¦ ¡hó(ÖWHU¥‘øátø„>'}¦¸²C¨½ÀÝAž[Dm ¢ßÅàëØa›¹Æ EUÊuÁµ¡2t*U.Š°l´†lÙ#Í5i­„,980Øh¤G>eÏkvèjIè¼[q8LôvÚ†@9VΓš6K‹´ƒ¼{ÔóY³Ce¦òÀÈÑÖÜÙÚj²+9Å(‰Öðj3SZýýµ>ÚôSjœ ­ÛT²GNßÿvý÷a¥{ä Í}Àc’ÖÒA”ò0Æ k¨cK° Êٜ޿ •^uH1s†.:B§ nÙ_6bÙhsÆíÃìðÒmÀ]88’AvP×öÈÍ¡š.¡`¼»&71â… H;1‚.&ˆb8P¨HóÓÚºaŒH ©2c®€Á…UÄ3´>}¾Ì3«¾ëû=ý¬ÏéÕìè'V…ˬc£œ% {Ñóˆ¹>+ó¢¤h…Þz{¨l·$tYù;Y®(D—‘Â`ñ÷ØYÑÞA|ývürá%IÛm•ž¬û¼÷ûj~vmâõE^¿¯¾zÓþËa‘]GpF× `wÌhÇoç1Äík;ðÎÞĽ}ëJýäxf~ÿéC÷Ûû'ƒ£²(|úÚ&°ÿ÷mIV’Ë÷ß¿V¤£mlc»´†Y~`—樬¶ šŽ ßžÁ`ïL1f@OHÄî<«@ÐŽP‡a!ÎN”@ÝŠ9ÓŒvF±,LˆyÁ×IŽ‰7íä'‚íšA¼n&;jÉZC6.]皨ÖÄ'Ž7>cON×¥©¼³Z„}3„'ƒvF5\›/`´(!ó¬o¾>*§U‰­&†QÛÄÚ (—â>’²Ú¥O$ŒEõ@ª[•y$b`p“0-teuìögmdÕ1†t³ÈY^Øï5: d†€³Ð(RB™lH`å£+åJ6`™A6£…©ìHÖJýYêWÏ}.y/:ýZë–ðÅ©sth‘´™ KSm¬½É„l¥ÉàáÎ<¡L²Â(ŒJú¼êã| ¿¦Ý4¾i¯gÌ"°4b6‰ô#ú †˜#üZ©w™Zúfœ3ÚÁPÀ ûË”'WáäÞ-¨¦—Ë´ÚÈOg¡kziéG¤Ä*Í—*„ȹ)ÞÃê²­1/1PÏÃ#g\]Wu^Í œ—-H²3¬È‘(]\½Í7ÿ?{ïúdÉqÜ‹eUwŸ÷Ìì @IÄë« äì"L‡¿ùOò#ˆ°þàOþ`ðCŽŽ°Â7d±®¯t-+SŠ”@âH€x/°ØÝÙÇ™9¯~T9º»2+³ºûÌÌb;‹9§OwuuUuUå/™É\m¤ ¢ q+;®T^‹‚”A¿h(ÒŸlI(êÑÜŸY)²Má= Åy²Ót•ÝöÛ¶,6]ÀG¨XocºœWq˜l,9ÑyO­ÅoZäâ_!ªÚçç4XšûðË'Z6¡Yd3$ج±ekC'àÕ–µè¼íxž>y”`Õ8ä…›ö…§¯™ïuµv]ä¾Èµˆ½.šœ¥Ai É@ÃS7öæî ~óù¥}÷öz÷ÓãøAÇE™ø×$iëocÌb4}i‚T÷ÒK/_r)`­”ƒÕ…É¢8=˜@±C4]ÂàÊ1DƒU½Æ² …tà D•k¾f‡ŽÀ­F,Fx¶S"[ÌÄÒEÖ†î¬ ·åõÀí¦Í#$Š©b=8ɶ Gñ bKv>ZÄý­4öøût³i«leðÖDÁiÃî0ƒ+«MÅ$Q1LÇm¾Ø`< ÀûE¸!c]xàDê¤x3wÅoÜ`œ0ãíÛÐÅÆ+Ì °Í™wðì6ÌÌ•…ŒH{Wí@⪆Ôõ «Ø#Z9ÅÖU7Øž!Ó <¯:ÅÈÒdé:¨ÃŽ¿S¾ýø#p©¦'F . >à ^'Ãb‚X<Kç)sK7šdŸîÌàÞ`(ú¶ðxW’P8ˆ÷å²q𥲖²ÙTÝ¥}`àªWx¦¬ÆàÛ…`/fB—#Gý  XrDPF¸â¨VGOÁâö °¹¦> €°¶!è)LôJó‚Ànìà‘xðæOÿÇK×î GÖƒh^XXÒ{¨y¦IRMÇ0œŽ*&‰"„!‚Œ"äÁp°úE§dŠ… €ŸœêŽÒZG*Ž†ð„--™EðØ6"<Ö¥$w!Ã+ÝŠVk»âœt)êá±ófmËÆò°À ÕEùoĆ Ì9.ׄÆ_¾ÑQÀB› dºe¡3ÿf|°*îÈ9‚°ž'M£.ÐŽŸ…ÙS^1ŽíðÛOÙoåšú8ÒWjÿWgXšCë.ÀŒ=ˆãØŒF#Øl6›äË.®o¬µvuzzÚG«í¥—^ž ‰óÂØe¬­S†4)€Iw _a°w ÉôT’1k°³fš1‡Ð¸ŒÊ’ËûÙæ*àÝdD<kÃ+Çž‰Û“’ò]ߟ[Ò);tb–Yƒ’J–ˆÖ{΀`{VO_+÷(ìê6FŠˆÂÖÚ¦ “ß3…±Zð<üž+‡ƒœ$ \IS¸²ÜT1I<{Á×H±`“Vù½‘p“Q~¥Wä=:DÀH;ÝCHí²Â”uq)*™@ûï&ƒåú§¤ø»¡‡LIøxÑŽ-‚™h5s¥)Ù#Ü\f‰!âõÙºLŸ„X(ªNóEe9ÒMw¸Ôp#¡ÔÕ,Èî´~èþœ‚ýîãÓÔӻոߩ‘˜ãl ˆ>¦kË4)ÃqŒĉÖx}“Ž².¹Ÿ„±‹<hüĆïö=ÓeÑÝÏnŸÊæŠcø¾Þƒ7†6 à±Å ™E8¹APäX<«ý]°Æ±<¨(ÅÀ`m)RF‰ñ:ºÇ¥xÒdRK’¶>ÌG“´ó¥zLò…ƒ#;©Nµ2ª¯ÄÔn  È XžBºN+€d0Ô›tÁ‘H,!’î %±]Ð/‹#•¼O¹âª‡zòxºábÂãŠlSðC@"1¸l‹MÒ‹$,óL†EG,‘6–Èy”泞¶óÎÃdÛù­‚H:4Œ@ÁFÂMŽ_ÇÅÆÁÊhLSr1rd[ÑF¨á#km¥àn]rV¬—P¶¹tuíÖzÀlã¿b_¼qE}7Òv‡Ø¿U Û‚·)÷›L±)Uª,÷ÊŽúîlb^úÖF}tïHýêƒ#}/ÖãÔ ‡&+Š $)Á’'EÎ rq4ÇÍ`0èÁ‘^zéå‰B§©RjUAUm–Hç_Ç°ÞìA>™B²{ ñô”ÎȪr.u··of*±½˜r2),_a;ÖÒð F¬eT 2ö ñ{¹7Ù0Üåz c‡(ù XñÀ˜±öjº­ ˆa¢„ò€-À,ù$¹?Âñ I2Ø]®a˜ ©xv2·¸ó¾5ô®~?À”=ÚXp Å6Ž¡•^€ Ü} ƒå»¢µØé²ÆÛ;²oÙzVè XUIÎxÆAY ”¦FLý5b@–ã_‰ÐÍ,¯Ü .háêÙ¬{nI¤r9|e½j°_‚¦8‚®4ì½®nZÃ|2§3ØD‘or˘,³T@”£Þcz&v—µõ¸16F¦‚miľ”un(È@aà¹2 B¨Ç~.@;«æM`qïiØ”©zéqì' Å .]Ûç¯F!E)«†±J¾~žyöZþÛ“Qôµ*MoãDÜȺï–o„øiŠ­Sž6 8ï0Ÿ¾+5àk£žÝÛ1/Ÿ®ôG܃·îÇ÷K€¤G0ËaéÁ¿R‚#Ë›7oöÙjz饗'BžÛYºya>™(—$Âgæ𦨩ã'#ÈWOA<]C²3‡h°¥ó:h¥•Y,ÛôCÈD°lI˜€K—˨났¨„û}€tÑi³´±RIøæÁÊ{‹ºˆí„j”Õ¾ž)Ê~C  épjldxûðŒ{üxà6dù'|@Ubl´†’ì®70Nsˆ!Åï‹ðIEqì7ß>ü8¬¢T4°,.„(Ïk—¢åò¤€XC³Š%¢`À;Ôí¢)N¢¬2ª“ƒ-Z{ Ż̸`šÝW¼KŽÖ–&0ó,ïþŒÿЭÛS±=˜Å¸/œÙ¯ Übø1ÿD¾û8ÓIJY’ÀƒñîŽÇºÈ´ —sú¢n"•w©ávòØ;뇇e<ðäŽi†v ‰ÑâJÇß#©Xªï#t1óïwù¿Òå¬ö¯C:9d¬pÕ)|ß Œ·9ވǂa€&e¯Af7“!xShýàÆ"»{.•|þé<yíµ×²ƒîVÌ8ŠšË‚Ê4¶ðØœ®áôÁG§°Y®¡¨,µ@“J•ž7/ªX˜¢— `©Öp[,nÕd¡&%8rY:(Œ+ÒË#¼ZÀŽðÚ°ìmçÁ†Jn—ÛÆ£futÕåQwnQնȄX‡°Тà7)ø|øèÂà#-äA~kÐBÕ‚QSq •¯ï%«»?Š³»–¤*8£/ºÆ‚lÛ~s]¤¬º9Q{/¥xù›O›?œŒÔs%!’ŸcE{ÈÆs[`Gwõ›>~NÚþ2  bYàö%I¤¯Í†æŸßœå/ꨞ+J6×p8„Édãñ¸ú•TÍ.jä"ÖÚ̳úÞ÷¾×ƒ#½ôÒË!åsy|‹í1•Sœê¥ƒ”ÀúØ<‚ìx ë;OÁúþSîB‘ «¬ ¨øPܶ>x÷ ¯ü⚎ÿ©@‰DQJù}/?ÆkFik¥Ð1·¯UPÁÝ‚«W#¾H‹"m¸ -«l½¤u°=6ˆ â¯SVe›A<é°MðZNýçR²JW›Û³ ÜÙ™ÀÑxiƒÑÈ Õ´óÆnï@²|o ™%>Ó@EÕBáFvU×IŽM®¿`ìEÐx]Ü»t b0ý.xWÎÊÁF¬&5s€@l_¦Ü’Ù180m²¢ûú½¨´Uó÷ÄÿƒàtþðŠ_”e H©ß Æcü{GoE_d·wvàÎd©Ör\[?ÂwŽ€cÅÄû}Ϥ`û}Àr½«cÕøÀ˜(¾jÛ#_8±9ÀÏ9nÓ •jgcØœ^…ŧ =Ó¶ZqRø²Tyë\Š^ÖfÒuOkàÙmeÉÚ3¿Œ¸Ç( VÙ±ŸÜq‚¥3ë?¶¦4mNV°:<…åÑ)¬OWu ×RiË1\”x_"LnŠ)žJ«1ħMÚ„ç)¶É6·.]î8ãÑUßmqE.â¦ó(®{RÆR`Skå¼ÈYWŒúGræhpø‘&·à$\Õ8…ÎÚLgó÷›þ~°ùøme–k\™–ÌmÇ9(ÒÆ ¯¹ˆle=ÕÏìÁ³/>cþݧ®¨ßc¸ê \Ÿ(MqÓ¨ä1#·HvoÆÄäÍ€&|×g%P»)`µ)cŽ”U¾¤²m$)ƒ·r ä I6‹ÅbõýïËöµ—^zéå’IQ«Rn±ÆƒÊ¥Ÿ¬EÑÿkd›û7a³Ò£+`Óqeaå©âÊÊu…3 …ð½¨Ø&0&A äyP_¦Žåû<¢ë_€(`Ïî ŠÕÁGaVO…e‡•ñë,w5Bc6×’8s–3hQm„˜q‰? àÁ•_¨$¹3Ãí NF°$P(¶U"Û¬’m…w¤³œ ƒ ÏÊÃŽñ¾R¬Ùž‚Š'#HÜhkÎnõÛ=)8Žy{€s/Q¢0 >ž y` %Ú ÌÖ%«†,Ô¤øw@ù>÷·‚3bcÌ`€3Š{f@Ep4™Â­]¸?B&öÍ 0`¥¥’ùx§OV‰UŠá&MÙ·‰å/¨Â^¨[A°A—<Ûvcý3D«Ø³è&;½jŠ¬oÀòÎ (–CçŽÇbµ Nm £PƒjþÄyA#9÷¨àþ^7W`Ì‹õúð2à8é}eÔ¨m=-ÇSø^*ªÍb ËãSX.«½„ˆ#zW CRÜ`Ô¨8Êo+Èu ÑïjH—e¿ Xè3Úϳ®?oÝ>/ù,¬„ÇÄ:Ogúä§CµzSC1¯ðõ¤y߀xâ¢Wæ nIqGÓ·@‘ýAA1²ƒ×ãÍG?Š³{Ÿ(•WÁŽº€¬mnWm€ÈYrÖ9ç ¼C>¾’,~we/ØI¸ð¸e[l.ÃMcËÔ á!¥¤ 8@¾¶y^ÀºŒƒTFÆ7ÆV€hS !P2›Íª¿%›äI¥Ô:ŽãKE¼—^zée›Xí—.xŠp«¶¡²a½SÎ÷¹†ìd›ƒ«°ºwÒÃë`Ò‰·®2eU*ɾԆA™M`6U ’hÆÆÅÊ«°#Sæ”PÒ[·w–ÅžC—^êOÙ§„W ©p#„燭Ãt{BÚøNÒ·!*”@Áwœ¥_vW€Ì²ByA)Ùlõ)Îío)‚6̽rL±gu=H¨g«l2àb‘(€°?Xs¦ƒîíìÀ­Ù¬ÊpdÚÝØv†W:|£dZD·  ÔâHö\¸ŠJŒåÇ{GD°Ì Ž}"˜†x€¨ª«VPcX<Ë{×!_%þ=Bfˆr©Ž|Nñß~vã‚€³°"ô¾ØšuDº·ûníIo–—Ñ÷x€€<`•™«’UJû@>~Ð ïJÉYç¢ iv,#rËùÒì¼Ë8ª(^™ ô€/ª_¤´¹©@‹òÚ–Q†Ÿ{–»KÈühc ðs·ßl»/—mM&0g—|Þ±JºÅc}ôîÐFçvxmm¦_Maô­ â«åt$ü€ùæظ#ñ[¿°_ŸKü¨iõ4ÆÚâ`°¾ûZ’úYj­°BWPU`Lð¼®1ÐvmS欔¾]ç*¥â£ÓÍÞz“F“Q³iÓÉâ˜mLƒ`¶´‘Ò’ÊÌcÀð¶tÜDµ ÿõ±4-™"Ô1ž-V•‰µ6çã8ó+I’Êõã“|rž,5Ðñþ”i|?øàƒµËZÓK/½ôòDˆÊó`ì¼ ROï.%‡PPAq0»­Ï©Rl–óaÃn5‚t3‚|1ƒdgñl:I"ûü0>†c0¸#¥Ð´þ3ÅÖaoè ½*-ûRù媡pµR™‚ÆþAÞ‹3¼QT´¾¥‚¸ ßL–%Œ¿Ú}¸è¸¿¤ÓÊ>ø5¿©Œ²ý¼XÏä¥qÃ&Òp2H`RÆ%Ù¤0NSˆ ã• Á2ž€¬‹e{†æcø}…7b±ã Ÿóx‹ ÚÏûFU s­÷hºA9‚ øx$6h *Ñj§\cvË‚¶rµÝª:ìžÅÖº¶À^€ÇÓ[tµf5×çkTð Ôú†–v ±ü­ñña°ÄÅx·'cX$ÈP…d©¤9.ÌÓãa›)–5JÄ2>`)²£üXÑ4üÅ lÿÏAEÁ~‘DÌ2CýÊßs¬ ;þVA¨Ó ,öo@z4[xɺ›HøG ‹ÌæÊp\׬<²5õmÈtïªçïýÃ_ž(õŸôàTqD†ʨS¥•‹#8ˆ€-Ôö,n PÇ^F"”Mš³½(Ðâe}'Z?)Z¥=€ä‹nèÈB-GpÁïŠIÒ¦„¶¹U´Ýû,àá<`ÊYc»Ê<Ïý¿ØB«|9ÔÅ*Q«;¹ürc¦Ï¯íä¥BÅ7ÊÄ'~yˆM‹O²·à17|-Ú,ߎWý,.æGÖd†Æs! IDATô«]ýÛ¨¥Toûý<¿uý­ÛÆê¢PvmrؤÌ “5ìî `g:€á òéÈ[¼Z6†|³ ¡”›Q~Þf“WÀˆ±l³eʬãzÕùðÏŒŒ’$)Á‘(IÓ´ Ê„+i ³UZoÔ-”T@a4ÜGÈ6Ce?ˆÁãz2Á@ÏlqV­wG¿m¡ ©nX´ÔûB‰«„A^K`Dû ”7$£²e ÃA84áãÇ'© áNÁ³1<±?L×K¤z.>ÆšŒ/ßžupàúü»?ÁÝÑ6:*$ûúÒcÈ »øƒ¸6¿ö©´}Ê„  ÏYïS g Ü¿œ%Ã÷¯D qÇÀX¾7øòËW ~–²õ.,îÞ„ìdèp]K]?£&„‹ƒ8ÕÜ&•lÖY¡‡[G¤ö]ÅT3½ÞŸêª£t1_Ã%”ÇŽ FÅ©Õp_U.GJýŒ¿Øœš½xÕúeü 3ÌWS)户Îó@Hü<¾h¨’L*æÈ&mP»âƒ´ ],“³À‰.°bØÖá¬ø"mu>¯ ̶2Ãç|Ì“G!¥6©”ÍX¢Í?Ôé{+»÷âF¿Q@|­šV*?e à‰M†å N0>ó¡CŸ‹ì(Nç?d½£a½ÞÖÞÛÀ±¶¶…üêꇶr¶ÉYŒˆ­VÊЗñ>yËM×°·3€«{#cZÌ,¡ô2Ò>ÿJ›»Z÷á×ðEaÕC»›”Ï]ºÜ”2 * ¤L üy‚$]Œ±60WJeq7@¶^z饗K-ÅèÔ*u?R‘±ÖhÜÔså„ÒÇÒsšÂiqZ(•µ}9€ÍêäÓ1 öOæ`MáNgì\× æŠàº®ÓצsE)ýC—Έáõ7¿ÀŸŒ·jcÈ°WÃ}2•C±@lìëb)Sm!T2|aþ7Áú ÁΆ qFk']È~¡” ­a1аŒ#˜ ‡pm³u º0‚…àÑŠFêI pÀ‰ Ô‹êçŒ×À~ý‚w¼¼ÊJõÀHeƸ`¬ÚÕø‚ë߬ӅQ?ªÇ®A&l¼>5€¢Ü;a)¡§dPVŬ/!°…ø„u@¤wÝñ$÷äØ_løzzC·¿ž$pwºwFC(DLÿ:s„é¬ìGz¿ŒÁ£FÔ,ë ‡ñØ@ív¥ø;k1¼a^Œ’¼SXl RCi¼6y ëÅ5XܾY»X¥xI<…¯·Â”CÔ•½Ÿx÷“R–]®|±\¯Ç* =Â,“Áø‹¡Q_P 8’®Óµ5p¨be¸rU/¤€ø Ê—G³×'‚Âz DLÂí“¥G;*f!R‰>ڧ햮 —(]Êcè žÖoçu•iºâ–„îmçn{¶.Pç¬úu]ó¸¥Jbµ9™ÁþOÇjð«\ùU1~Ù€ÚÜ° NÞÿX¦8“´Òú/"äe”b[ä÷ëOþ.6Ÿ|\®x¦Åu&3q»º˨ L Ëjëÿð­ÍMd±qmfî®a~’VLI¬t©áþÇ´Öû¶”àIíJ“Cž#%±ÞÔbô÷ÊyɘüQ„l*™$ŸA’’QòY¥ ár¶1f½³³s)®^z饗.¥ëµ8´ª¢€xë4#Âtà…›ëFaLd˜8E$?C¾C<›B2;…d6'Wä1 úõëZ®–íS“††‘!W–¶ø \ p†ÿâ^ý³sEYQ&ÕÅëµ ¡ŒKG°Š&5H²ÚÀl³UpÂJŠ¡Ìö)¾%{Dråëž0Öœ­à@Ï‹ 6‚xžñdÁÎ6 ‡æCÝêT½îù5iì©8ã³ßÔÃÛ(ãŒÍ,ŵô¼dŽ4†#áÈônÍÁÐu ‰d1¸wÆmQH_T¶r)Ú$ |:ۃᰊ-¢ ¹Ï³FŽQ×ÂÌ]„3‚˜k ÔXQ žÊ\oĘ €@bpð¼Ç¸Âž¬³Ø(ÑO¡»¸² f+"X< Ë{{%·Ý×™¡¶RlŒ•.†J¡QÐ3>”åÀ˜0â»o{éÂdé]Rà‚Ì£ôb,ì/J 8²s}ÇX›Y3(›äDt] ƒ ¨‘›p`O1>—N¸¤@ã8âLH0hˆ?æWJ¶1)¶Å÷8«ü.P#¬Cøù¼÷Ùfuî¢ì_Ä #tê:Ÿ+íg8C”²eÒ¬“¸ûãhøÖÚì}k ã—3]+ýÐzCè2ß yÐÜ/,¥9,_½7X¼÷£=:„Hmm/ÞngŸ6ë< ”¶¶oëŸó€XÊeŨa9c(ÿ&7°°ã“ v§1\¹2‚É8‹FÈ*v?ÜL•u˲ŠÂÐ|_¹ÔV «ò¼ÐÔÝÙ$èrSþË«”åöW»z^ìKYEu±Xôn5½ôÒË%v®›¯dÅQ4( È~áÄ4§Â’> ÆÅ©ÕV©Pƒ¼-ßXÈç#(NÇíÌ`¸7‡ht|Q¨xj¯€Nà ’zíO‰á,hÔÐTèøµŒòîu3À[#¹—æêŠt_©eãueüÌhs‹QÖïÿ-íƒ ãñ7\rHán©HÑòY×î0²®(r¿oLËæŒ*È&«R„Ó=œ÷x¤ëýs½Ù+—û¹€’`*à,iÛv]Ûö¹MÚ\ãx_cò2æÈt:½t)Öz饗^¶É¯þ‹õz~ç®] (Qé‚ 'qãA‡Ò…P0 mìEŒ w?T´¼TA~<†å'OÁjÿ+`6S¯g±à«• rf[á‚ø 3¹ò˜$´_ gK@q´i3P†.më ì?ÜJ£‚ƒŠ¬ÿPÀ×Q¶™¯îÃë~Ѽº¡£†t‘¥L)Š”9ÞÖÄ„l ÏÀU,6µ’’{®2.É|˜À‡;S¸½7ƒ“ɸ:f9 Â®S j8vœK‹åc© ˜b ÏVŒó€–)ßž=®6ú¼Bâí8Ààú÷Ái±‹|l Œ¥x­_§DãxÒ>X1°!îûܹËp#9s½2®¯õCù>ŸŒGðq ŒÄ±F|;%Ç‚+Œ15Èð°ô0µ;~'€¥|&è‰ý%7™»M¡D¨µÛ¬Ll(ìzßx»›z¾2¥;XÙ6ÅVž†•cŒø@ºZ¸ñx€C4Æ!žaœ’m0Ä—‹1Ö·ØU·VkcSc‹ã«Ð»Õ±QtÈg(1áÓ¤o½;ŒÇÙ‰–fÝ|âôeX|,9qr´Z £‰”VEƒ;ô™…]mG &DÊDOÍìÕXÙèþ2:Zg&-ÿ±->ÈEÀ–mÒbœxi»ÏÃÄ¢¸H,‹®X —Ub•.cxðÖ0Š>ÚؽçÖjòrñÓÖVï#M¯Üg·¦1–zyvªÓƒ7†Ë÷þ­2Ë5ND],¤.÷—mîR]€H›{MÛ=Úî×U^[Ÿ•#¥­ Ê}Zk¼¥Å¿ã‡')Ÿ¦0FpuoXo "oà“6+¤dŠyÖbùaK'.¤r[òK?GáL’²mÊÀ­å?Ö³‡—¶÷„·é¦µNoÞ¼Ùƒ#½ôÒË'*Šuª”­Ü9?{²S;ëwQ¯¯ÌxŽÖc@»1 4௚a‚‰ÊäG30Ë1Ä{'LA¹LèÕš®ý>µuÇò1N„–o¯(Aø›(F¦‡UB[“®ºþ|:[0€µ‡|Š?)¹WéÂ{x9þÆ~NÏÊEÉ׫åZª¨¢2 (ƒ×‹XA¼ :…)ÔeBÃAGƒvFƒ*»Ít“BR˜&JB‹¢løÈ"اò rKñ>­Ün¬¡€¬5@ÀÜ0Äó†UP„ÂïÖºØ%%˜æû”¬ü4îä~Èß›¦iL#€\ J‰mx­X}kÜÒÇ3A®š5ïG]¯\k8áöt §:ò¬Æšâ¶5oìÂû2¶§‚¾ ƒ *P" wÏ¡9ƒ#†¢B<Âz®‚!<Ä¥:P°½©ÒPdcX=¸ «;‰@9p´n;ÍFºÑiV¸´±ö$@<¸RWß»YÆäñ€%ø®®%œ£¡Xç—2®ÝcGbe-¸H-Ø°£00˜XÇl(7øµåÊàˆp0¡¶¦ôû*ƒ›šF±Òæ‚#ÃÄhÐ{c»óµköÛ×vÌoÆF«ÔÞþø@¿}ÿöW, «‹óÆÙs:€–ðü°þ]Êr[}¿]×]´¶Ýó²K Å*ŽÞÛù­µÝ}neÇßÎ!y4 ­)È<ê^åt±9ˆ×w~šln¿¯ìZ0ÎT…íÞZðcçeïlcµÝ+{ e½ŒkkËÉÁ%‚Õ1”šm’ÊÃj]Àj³„ñ`SÅ%ÙÕ ‰ÖlÞ¯6´ c Ïr±Ú㢛`\+À”É,|!ºfYnFeôu“¤t¹s%Û€ÂmïIŒÕZ»üÞ÷¾g^}õÕÏø½ôÒK/_¬h«ŽKg¨À^Txæ(Iän¬-\ÅU8P^1`g–?ifò“2³Í¢ñ)è$c1Lšsv®Ã°$ [ÿT sÁÇ0ì”\&àlì‹CÀ£0€7¯s EÆ÷À6³ `aIC×ö–û2ô§fàgiÀ?+Wh=Û äUÞÉ?¯„1<Èà2ó$†Ó$Éh{k’” 7â3M\°ã8à25m0“¬Ÿö}(XÎñ*Fx|r×n€[ J»OEKŒ#w‚mÓûݳ"C_1%È¥Í6êo©O£u2̦‚Ïa+_Èâ§SØ`]2¿˜.g]ú]CyßUlqÇ<9j‰ÀeÆ3ÇþCO@‡\à\1DkÅÇ'á±7}]xcߦ@q ä› ¬î߀õá¬â5Ó< è Za\`U–ÚZbÐY¢®Hˆßf³¸7È´cè¦ _6å¡äª.u€ìõf“|ík«âwàÒÉcG”|ªÀæbÀÀX˜40ÛQ57gËÕ¯¢5Peö•ûY0êÌhu™16=/ý,iSôϲ²+[¨ÉPnLÌÓÏ^3¿1ÁsZA$v24/¼ð4|õ¹ëêîݹzÿþI¾¿LãÃU^Ö»;@æyXmç·Y÷Ãrº˜­ op¿P.;ããó’2 ðT¾5R§­ÌèÙÜŽŸËôà¦1ñ®XÈ×Ñfñ~²¾ýFœíï䢳C°a¨±­/Ú˜AáxÏ?kl»®Íµ%VjŽ9ÂàËÆ|1eÖ2¥« ö Æ&©Ü:6[çqoj“¹ÿþêûßÿ¾}å•W>¯G꥗^zù|d0øÔZ•3$¡^G4¥‘ÂvRž˜¢Œ¿ymÊ»œ©¯Š®/VC0›!èñ âé)Ä£èÁ”ö¸º¿×I™‚Íö°á^Ù€®¬ñ{i$ ‚’e°NxpYT:yÀOGÁïÙ}µ”¿{¨L+¶gBê•Ûü“’Ì›]#BH3H+ÏnàìQãF»sE¹TìªÀ­³1ŒG ì¤L7 ò¢&e£Á"¡X-n=a__hé ·b1dÂ4¸a|’ ¥~—â—ÂH°§’D›á»ÜõiЂb@µSI™ah9½éæƒäô®øX áP#  ËbL^'¼Ø²þ ˆI’ùÃç²|Ë5Ñ#Kåʹ‘ZkåóRPã  qìW÷*²õ¬öoB:¹²]à_«d]h €/Ôú>²¢î®jˆ 0ͳ ¼ËŸ–@1s{kœ¬Z/–ûGÿÑ%5À=6pÄhõÀÖø–ÁĈv¨½þZB?­G3‚Íè X½¬zÒâ(›ç䟮~e3ûÐVáóúý·)°ÃØ&OMíS7÷Ì·®ŒísI WÈFÈ£Öv<Úç¿~žyæŠ:x0/nÝ_ÚOOõÝefRèP6CÅt›’ü0–fhQ–ÛîÛK»Tn *]N£ôk—·2\Mõè«Öè+Ér¹Ÿ¤ûïéâèô<Í· €;+Mø½irVùgamÀ¿®Êˆ¬Lì\Á=r†±…ûus‹Wyx½)`½1U†›2hëÞN “ú`²ùÀJ„œ#ûÞ´¡²>_·šmÂÙ$¥ÛM î\4€ëY¬/kmvzzºT!u§—^zéå «ÌÀØP\*_‹·pÂs<Äá­·ì4:Ë(u°U` _<Êô¿CHW äÈ'KÐãU”€Î¼2É€ +6Â-z-ß&,ª`CñWž…í²è„qGjߪøXY¬d¶x ¼åÊbu[+ÊhÙ듵¹imoˆh?0P¶Û.e4ªÃ,cZŠHæ}oõ/¬…E’Àªds0M3˜e9 ³"ÃTáräXÀÌUÈÇ#ñ¬$ÎBð:ºHáÛN؉|îTîÒCL:Oƒ]5œ¶ÄÒ¯¶¿š•+\Nˆ•€cŠA+!ØH­ï¹@"Ñ?þˆø4[d<†ƒñN£ˆÀ…¶ =„©rÛACïÚÕþÞÑØFTŠÅfN4 sáâ o>V ˶¨dë\„¿¹2ŒI [ìÁòÞuÈ–áúÂûˆ}ŠA!ŒIC<îÈA±sðzÊáÁ,«b&`­Èr,Y³ÜïÍ£WþåŸ^J×íÇŽÀÚîC¡UOÓ€±’Š„‘Ô=r¯ñÁ¦ü"cÒÚ,ƒÙ(cYóì½ìîòC{`î‹lý°ÄPùèR@+%Œ¾6µ{Oí™o_Ÿ™o¸^¦2A¬5Ò0Œ”}晫æÆÍ]ûâñÒ~²ªß»{¢?]¥&ƒ¥§ ¬8ëûEž?”ó–Õ³IjQ*ß ¿“¨ô)–½6eÓG4d{œåBs–›Íybž„ç·¯çÝXLðÀwUÜ÷’}§ÝDËFÇ]›få¿ hUÀhPgíçzþ®aÔ|pÌ’rZÏ ¬ŠÖ%ù‹LEQõü%@R%çI¼í½²Ö¦Q=ô\ØK/½ôò8Åæ>(;POS€zv#çaw¿>£›bk Ói…²ÊIÊ¥·-×b•€Ùì‚:B>ZC<[B<>P…P@€öyž!AUzU ¤ŠµŽ5<3Ö¬ Œ û WrH©±—æ €X_UÀVòç¡pÀDZŸy°Yh[UÃ6¦úpigÀÖL¤cjP­Uû¹a¿‚û­\aqë8‚ya`še0É ˜¦)Ä,.‰¨WÐ'rì)¤°OââÀò‚´ç @³vÆ s»qÛ™ÀûDÞ–= H -V¼ö6ŸÆTD¨ú+?d±±_–‚åp÷&S8$²^m~_÷ÌÚ·‘ŠZÚ{+÷˜–gÙd}4|\(a1&oÞ+˜‚Ä–@6 PäCØÝ€ÍÑ.ëØ6á»PýÏx¥XÑY,h0›Å94.¨ŠVŒtâì&Á£‘KØxAq —T8Å›C˜`§j ‰úº5ebej8ÿÖ‹·‹¹×à 19™!k#áT÷dQ¬Š³»«_æŸn>¶›bcó‡3ÒG¡ís(;‰={Å>ws§øÎxו‚QèèÑÙ`AªÛÒx|õÚì]™šo|eÏÞº} ~q{®îæƘ.Å·+æÿ}0q‘€©á}."ŸgÙ—]äYe”Ћ=ÿyc…tešáºØt1¶xmû¾µ>õ7M¼qAfomÄFŽ­´lPïûpʯ'pM‹<æ÷ÊÄâV–::º-œk[psÎã ö\‚$Ë&aýP^°.åÒ<`/½ôÒË$ÍFscÕ]­Ô e”I«¼k¦Õš41Š’áX €1½§SŒš® |)°âžéœŽb61Øl ùb ñt ÉÎ èÑ¢TwüyVør%Þï¤ÇmYŸü’ªä5JÞåò±!¬Wƒ]¢$ bƒ¥¿k 2dF°ÊX% ÏicˈX+LÑ “2Ð1ŠM"Ýj}+¦ìYêoî²SîWÖ‘†M4„ùÀÂx˜À,Ïa¶N!)cM2Du%)Üïûñ¥Ä³0w&dž“9P¢\œ4<7u›+»\꣨Ž1á]˜ËÙ||)¿›"e™1¢“|µýÞ ë„_y?0G‘†£ÉîŽÆ°Œ#0´Ç“ýÛtñÏ)]0戥6 Ù6šƒ‹tKÖWJy6Q ä™ô‰ú…îÝ’šÇ ï*†œ¨² ¼:…åþ H'UF`€F7ÄGƒ% E2£0vL€6"hDY¡ ÔÑt5ƒïxl! ç¡> ‹µ*îuOŸhPs¸¤òØÀ‘›‹›fy°|-J¢È\Sß0Sý‚¨¯—q°Ã(:0‚r†Ñx:¬òxÄ×1%']>é•Gs{Z̳·²–¿0óìä³€"ðicc}sÇ^yîFñÝ ¼T\E€‡§^2ÝV–pU¦ÑØî\›Ù—wFêëÏ\÷?9Ôo~2/hU-p–KE[¼‘‹Æ ;«œ.ÙǤíد»ËNWŒ™¶óà‚é’šnm|»Æ×yúG2šÊì~*j !4|ŒóÇx‹Òî“IÜBþª~Q¶”þ × oyÐ67ja£[Í6is¹ic“l7£(²EQ¬©Ýôz饗^ž4äöàŽª2|j…ÖdåMÉ‚çd•ÁWÑÀëW>#-?U)†™ó™Â*X+5$`Œe4äs’Ì–0Ø;=Xz%†k\tofžl«Â¯ã@‚PÆŒõk¦4à1Eh_ ²À®­!#„[ÊËŒ+˜ÍÄcx%gæù, ¸Â/Á j›.Û­éÞõ¾-HÈDE ;¼rYÆ$É" ¹N`‘Äp8ÀÞ&«²Ü 2f¤PLá&e–:ž/®OÓ ã–ÔUSÇ[ €*.÷àƒ X"¦~C„bŒåk¬¯U,°iبø<Ê+;‰· ˜^WWÞS£F¢*èêíñ6Zñ+Äë„V°D\sñz*^3O1!  tjÛcúŽ§gàCKøFñ~¥v ÆKã‚ˆâ  ùú:,î\‡b™Tß$¨`h<¶/fx…›ËÀ·ˆ‹ü=ÆÞõc­ ÄêÀ -á–?U¿!ã×Ø sdÀ.ÑtY$z\õxóÍ7‹çŸ{þ¹XÅ_Õ 8€¹ýPYûz ¦|•QÊcˆÁïñ0®<ÁTC!`µ:¿0iñiþñêÇÙ»‹_˜Ó|E9¥RJ%¤+€«®SbÒ_”a¬’oÞ°Ï}ó©âߟշ#]Ýdq°hâWô]~†ÆgöÞ–Í5œ ÕSWwŠo<=ƒ©ÍõÉIªÖˆ†ôÁ ý6¦±H`ÝQðüË•àðxˆØ·] AÙü¾üü¶ëºÚþ×Iʶ*¼ÑÍ"‹–û¼²ãü{ÉÐí01_رßÚÆï϶qä«Ç*}.‰ò›ªž…y;„@}ï`/H@‰â¨¶3³IÃ2Å/¾[ô 7šî\Ívv5@›Ÿ¬ã÷Vúþebås"“¤üW~G€D–‹àÝ.Oz_kýæÇüÈݹz饗^>oùôÓײ¯|óŸÿ{I2þ½ÊÜÊæE¾bÔŒ´¾2E­Æ˜AqU3Éy;o½Õ¾ Î|¶¨ à]PkÐuŽÍŠEiЮ6 IDAT‚¤PMÁbéaÀ‡ÏpQÚ§ÊÙ`Ý 3uH—…JÝB–„Ê·róct>)JJxeŽï+iOŠè•SØ8ØBd`%h؜܀Å'×+½0õ9ŒQ¾)Ey€þ°±cSNqƒ<2=ÈSƒ_ªøtáúC…+Ùl¼éëK2 öïOæ·þìο¼” åÇs¤H'Ö ØÀ=ø•šÛOìMý›v¿‘ÞSZ‘ ·Bó0ʸò ˜å_xչʨlR³ÈßM?\ü,¿³yðYëŽ4ömZÔ•µjw¤FÏßÈ_|j~;Š`/œê¼Eh4ñw­)Êdüæ²v%‘Þ½2ƒßÙe/>·R¿úàAôæí9AGì‘6IWÕEp^æÆÅ=#伶W[ìò_9nùg”®þ>Op×®Ø%],Ÿm “êoùó¢zOê7vÐÆ/ãˆ0ßZèþ³e†›–źÃRS[ØÊ i( µyTýöE fº‡‹S‡Œ#–•OÎçóK™¾—^zéåœr”5õ¾Q;k&Wy& \kÜúTE7NáV eÀ–Á7+Sµ©ÃêÎBdQM "²‡X¯’M’EìB~:dï’#Ðqæ,à^y#$£ÈR/oÝf +Ó?A`XÆ\÷¬ÊÐN¼¨ò\ñ€Úk~¯h¡‚ó¬<2KøýBic«„†C¬>¡Ñ§‘†OÇ#˜—þï› ÌÖ$EļÁFðL¿ß‘çk™›Fêªh1ι5¿t«©±î¾#Yé–Æ Ç"|² rÅqO£Yvïæå;ƒž&d3¹ºeƒÜÝÙýÑ°JÛëÎÄò.°|¬á8ÇÑËcöРµ‚}/¥uìoô@†×Á8KÙ³lpÈq § ÒLŒgßît jÖ²Ç`”wíãc®š’XÝ„Õݽ*Uxý›34b¦wMnWÊSÈ\Ì|ÎÐŽ˜+ç®C£ÓÏ Ü]HùëæBüÊX>Œiå’… ì2:).-;ù±‚#pªµ.Œ1žÁ²6K}Ûþ fpË\µÿ &Ñ VôGઈڅ%§ê÷¸e@!0’›£|óÓôýÅÛvYdŸ¥ÂçuÝS¬-ÔW÷àú7®ç¿½3Q/€¶ ÐDÄÁÕ˜ ÿµ €wT\W±–4[°ªø.°³·cçŸMŠo?s oþòÓøŸ.h+´€"á1~^˜u¶(Ámל%ç‰MÑK»tÅáýÕ d›ËUgàÔŽ2ºÜnºÆ cI•ÞáÚêƇ6+ σÏ'ãæ&‚ï ´›ìq£,\ðpÁ¦T~#eê-ñënR¶o  ËM ’”` °>°ÖQ­nÞ¼y)]‡z饗^Î#ÊÂ}°&¥ÂØ$̘Õ"%˜ÕÄW³¾µ§ T¿!©QyåUc™5Uh ª¿eúCeÿËØì_l>…ÁÕˆ'Ç ãÜÌÊxÊ3>-*z¦fÀXö,¤Ô–~R¾w߯s¡E¬ Ãö˜ãƒ=CP€¦Béë¯Gª½en8À® $:xðuÏ(m©=ºdÅ +·â@@“-Ë/ºaÀ®íbœ”Á[—ñ&£®­60Û¤±˜$8x-;È6¾Y[û, ¾(ÏáWÔ @“Ä Që”Ù&˜/]Pœ –Ržaâ RrLiÎò ÇÕÛ)¯ÛŒpg6ƒƒa_ÄXqüëäû›?”´×kkWßtM£9° ´Çd@ î95s%{Ká´06øMƒ6á±mÄ\T$°>º‹{Wòع}—ï`ä€ ŸuQ±±j Rǃq{»r/ŽÀ #uÛ¹sÀ—â¸ÅòãI:˜9ÖT¿Õ1NXêdðó„öï_žêøhý­Q¯Á¥”ǪZkçÖ6ã}ØÂ{l÷Õ'ÅßÁƒü¯lƒÜ:Î’"ÊúÁ™6—Ò±3-éÛëwOþ|óÆü/@Ñ,½±.†/Å|ëÅg͸3…—Ë ªáä–¼H°²„•¼ùJÓr-ëЇÒÕlo¿1ŠÍŸå,¥uÛ³…ÇP9çÇ·1¸ð:œ7 èyë÷ë"Û˜#]möyWÆ™³À“‹#îÊhCÇˬÆ¥ý’.¨—¡¥ø‚ÈPjn1 ¦ºÿð]”å—¡Y‚Ërm ¦‰*3ÂØ'ÞÕ¤ì¯ ™L&0›Í*ÀÛ¸¬W«Õê{ßû^ŽôÒK/O¬( wÊPXuë霭LKtÜ@‡%¬S:¸%ÝÁÂB k½€,ÉdnfJÏRn]ß¹ «»_…ìô Øʆ¨ÄþŽ”ydT› =o0²:îÁØòé5Gn³ãtÿ $ÌfiãñKHI$ZºìP!h›æ`·¶³½-ψ#âïñKm{ÜÁÙA$Qn\ÎBŠM¢8èão$3U¸‹DytEðñl\ÙÃÙ²$’»z®€óî Ú8É^A¥·%£#ŽR+¼M†€b[ KÂb)!|¾Ö=cý*ñCýZÁb4‚[{{°?:§Þ†^ÏmàÊ4¼ëÂ@íâ^4ÃëÈR ³@ÊB´w£©ÜŠ4PÊ–zŒ1€Dì=ùw¿¯$& z³9°Ñƒ’îý*ç¢<†õáuXÝÛHK°¨ðó‘ï÷˜Úµ¦[mZÔూ|>EÏíÚÅõ?ƒ1DX›ñÀ²(*ç$ƒùˆ‹ÏŒn‰$Q*Sùúèæþw.íó±2G”R§exÔÎ2ب»æu’߆=xÞ^I~ buMiˆMmîe(0ÐKZfc7ÅýìÞú§ÅíÍm³È?³¥·Ëf›"_#ÏîØ›Ï?ek6²/)ÿ£`1ê’œTä¼ËÁþ»CáyNyo(Ûò–®E%›Dªkæ<Ùj¶CíR´»ä¼î5•_WÆÉYL¶¾l=ºôB0&Bà+Ø8‹åL7šF}Ë(SEÜ@ÖÙÆÖòƒÐú¥ájÃç ±'cï]‹†õúý,9kŸ d½L‚ïo ” ÌpS-_yå•ö]X/½ôÒË F™}£*µ‚o)ïlÞwñÚ•¦V ¬s™ñ{Ko…­•UKKT­„k÷;SD1¨(DÇ÷ŒT&¤µ(0ˬV7 žM)h«ÒEÓ*Íÿ c¶dX€ QX ±MùBÝ!U5c—TŒe–­FydûQvOÏñ ïæãc–PÜ’Ð-†p&àÒþ$XÀ¿‡ìÑ6¬A}{Áf™H0‚)Æܽ¢Á ©ë±Œ4¬¦G°·Iaw¹†¤0 Ãº2F€ån5Ø·àuÏ‚qÓ*kMƒÅ1”HÉe -V°Îx½,Š¨itpU“ÚT4^u^¦#8žLàî¸ÌH€€ ˆL(ª"c#à0âÞ\ªÙ»©x;)ñ¶Ø;뫇€Sí7X£ÛˆŽtslà˜ÒAðZÁÈa}‚দœ3`Š!lŽ¯Ãêþ(Ò ¦fÅÜ@·%ëæ,C±C<&hÙý€µ”Ï8Zœ§ 1Q™r  ¶ºeT}¥Â^öÞe°P‡ßû˜W_…K)I’d¥”2]H(ÉÒ,a oê“ì–Ýž·;úyØ›6Ò£&3¥ÿžM*ÅÅäÅIq½•Ý]½]ÜM?seήèRöº2y c;zjgý‡£8~ÖR„i0l Z` È5έÿê ¢9ÑúÜ1ãÐý<7D‡Ó`õ ®«óyÛž%dœis¹8K.êŠÓËÙ1ZºâÅÀ–I8Þ»þv•Ñ‹$¬Oø›R•KMb}Î7šn¹ÅŒÇ 7BbÿÇ6CèWªµK_G¾²Ü7V¬&´Qªý{•ÍŒúR)-à8ŽÓ¢(>Éóüãz꥗^zy’$Š£#ÊÅNò*€ÀA鲩h=ññIP7S¡bIñLœË]Ù=‹¤^VüùˆÖk2’aŠ[ŒQ×0Ÿ¡X!ÙYA²3¯A—ºÈºï•2Då6`–0@nÇÄUe¾Ae•9r'áL¦”…Ÿ¥ ä`Áúý±riŠËuÚ¥û¥{·Ë[‰c…H† °=¯»äCÖ7`oˆ ½LˆA쬵†u“dT ÉN™¸I¬mÜ“À¢†°ÌI¨ôŠ€™™øP÷cJ#Š@ê*åY¶Õ0@Æ­e'ÖÀ>¢hÍÈ ,B×µÍ`÷¦38( 3øÎÑ —VxT| ±>Àú¼_i†*C‘úeÙ›VÞd‚)ÇüP”I†=ar Û…E€ãk€m}c(Ò1¬oÂúÁØBɵ.ãg«©p¯ê*¬ÈÝJ¶NèˆB…—}„xWD*!«…@4×N¾[«ë—¦0?Û¿ýæÏ^ùßþ—ž9Ò&EQ,“$)Ù#»ç9ßžš85?W3ýA1ƒ¯«øy5ÒOµc›ëb‘œÝKß.înnÛõççB³MéãŸmaíÑ<æi£a ÓIIUyÅA ´–r#•Í NZý«}a-{i¤+MY¶D7kÄ]ÈÜ6.&a›­¢6“1̇ØI3˜nRæDf{Ù‚½"~ñÆR¤Vf‘úgÙÖî6$£,5ÐlïÀàkˆqÃØ-<&(¸L>£ìO¦p#I˜–âfØÎZ2q€ÿ,A vZCÌ€á~Ø}-sk ¡ q&‰Õ•@ x½ ÷“˜i'¨ƒ7nǯ¦°º²ù¨bbxÊŒ=€ {äÅO}È÷N¹|&œy¤‚ÎS h®©]ªX™Üc¼ðæA·!†Ìã¤ÞŒ—sÁ±µæOï¼ý“ÿö?þ¾ùóÿ\Zy¬àÈÞÞ^¶\.W]ˆp—” I¼Ô¿°óü–™©¯¹þŠ-̃âîú}{bÖœ1›l‘ТÝå~Ðæ¢Jº°6«M›ÔÀr­a<Œ`:‰«4¢Zˆ ðɇQÖ {å)çç”×”n4EéF#h*õYZU±¼’ín gËy¯ÛÆ(éAÏ.]ïÎyÛö¼ÙhºÎï ƺ-¦ÌYuÕ%|)¥.,â…VbÈ3 þ´P9‚s9гÜòæ]ŒQ«ÅJ½wëxðóÓ \ÊÔc+ªFcçyžÿão¼ñwÿôOÿtïÉ|’^zé¥/Å$[€c©+wXõõs_A(~áAêz­1Öc€’{3 Š]oQ¹ã(%ê ì³I#HÓèå¸G¢é¢Ñt¼aþ%ì™J –L^E¯ì’†ëoS« ì`…y:¿70´…Ô©1†ó‘° · Ö{Ò·ƒÌ" ˆ€Ü€È]meá~Ë,ÝqÛâˆxÃ&ûÙ–×–w_“çŒÕRŽ¤eÁzÁ|0€IžÃ4Í`˜æ0,XðVAÅh!q050¡sx1@ ²lìÒ›`™jL:Hù/hS¬Fš$ð`2†ƒÑVZû>£Ùx¢Ã,Ạe#óA©`@p݉$ Xc1=Ê‚øê]–pIñM1ö%°¾/|û±­Ø`íãêgŠ6§Waýàd‹Ä×Qr¯Û‹WÒ³ˆ0ëzƒÏê+GâÁ-%úÄÏ-¾lÃjÅMÿÖ•–ÐØPÅÿò'ñ?ýòÇÿúí]b`78òï|'íµ×Žu5%²„“h­ß*NÖš2“™ÏDwoc‹t)ƒmVïF–ŽÒ[NëM9.ªI!3åJ°¤If“†CMþ{~1åH´ 0Àz¸q¦³*ï—qE |ùå‚àŽ)dŽœP|–˜!a›]$ÉY×öòÅÈ6P¤ËÍ*쯮ïÛÎÑÖ(Ót\$ñV)¹øµùö†tJ´z)¾Ð“%J¸ºâÚ`L¡Oêç·ïÌ7êôË4üJ`Äsßóÿýä'?ùÉ[o½ur ªÕK/½ôò™å[£ÃÍ|ï–Ï=€V"ÍÖ¦½[ÛX¸ç=Њ¾Òêíá¡M‘ CSA“û_,“ÿì2ª $±éòõt’A<]TŒ=X±²†E¸´’Ù;`xp¦0çkåÙì°Çm`:äºà”\7a;—ý‡/mç´]ßÕGá5x]xŸð~eý: ŽSÞ–gõ ¸ëp 㘠ËÂ>âïBøïc~œÿ~çã¡1ž@eÊêuyffªÀvƒ¸Â¯˜m2ÚVn'I¬a2‰=Rnù–ȯÍ”I3u÷Îqü·Žâw—™^}ɆW¡µþXkýçû·û³wß}wy êÔK/½ôòHäÍ7ß´ÏýæüžRð¨)¨`ó¤HÉgë…RÒ,… 2Sx܉x:o½Úræpó³W&¨ÍU©:!ÝQ‰Ïµr¨Kš2Ø,³A¾ƒ5 è¸É}[µD Cì19;d±ïôe Žãóòòƒ{q%Ž‡mì⣴‚,¢ë¤[aÑëÝnðçY)¢ZömûÔð>¢Á}ù•m@I¡¤eðÖ$†ÓAyW]KnÃ|t`Æî·‚(oBÓÞÎ× Á…òØx\rb¨òsý{ V‹Q¢5œŒFðélGƒdŒ“ÁÅøp`u=‰ Q¬Cˆå‚¿yL±gãÃO±ûÀ>?~³²«qí kâÈñ{sùNY«!ßìÀrÿ)HvÁd1ƒØ‚±@¯"¦H›xˆDáKèÞ:Í®( ÛÆÍK™P®emãžÃ·!gÊ÷Iý€©ÕöõÈèÿâo^ýïÿÛ¿xý µÒ—P+sjeìØe},ô€m±EÎ 2Ù–ºV‰Qz‰þo "€,+àøÄÀé2ƒÙ$†ÝÙâ˜G%¶â ë|Kš’1â.q“‹Žïc=AØH̤ۤcƒiè^Ó?b[VžódžáîÛ\,zyxÙ–Y¨«/ çg1Døçpœl‹C~ǃ¸èâÄŒÞ=1Ã[SÞœDë—bÈž‹´¶=“eÜ®M ÿ®£ðzoíɾFÖ©úàÃø×ú 0ºø2 MSòa´~'Ïó}ûöí>øàƒ/eÙ^zéå×\Œ¹­TœƒVq †{¸å™e8c¤Á&Ñ”ÂÈb«½…ƒFj%â\¡…—bh Ø%dÚç ¨c,†¤a å¼ŽeÜšUiÒUúO³¹ù¢ÌnsñäT”K¶¥åzšb%[Êÿ{}Ì3D̤ìe€ 1 l;à"”Ò F^ƒ]Ò@Kü½šn/’ýáÏñFžqÒV®¼¶)-@I¨" ®òGÖµVŸ‚,ÒŽ†p\Æ%É2Ø[­a”:& ëÛ0â„eíÆY-\_(‡²Fp‚Åá@ÓгIV[±qck œLFðét ·ù²ÈÐpéq}ùr|qDzfÙP±>; (¯ïxÏë ÿÁP¼ÑŒbí°´>[¹ôhö.YÆÚ ³æ@=ùäË=XܹùjPÇÁXD  à@ê„H]asŠ›ð}÷ZK¹Û‘Açæ¡@.“j ŠHelžsý›Z­~`}å­¿ùþáþ[o}欱_¤µc¬ÑZ¯O‹áÇ‹|p{ ó+;ÑêùÎ^Š´¹JóE°8ŸÇEhµá¦…\ÄUqºT¿ºuÿäpZò>Ÿx)‘_ ƒ?}î¹çîüñÿñ¥ÞK/½ôòYÄ‚}¿ÔCÀB\§Àh©òì» ¨ñÁàJxÞºW(€­35¨¢Rfd¢¸½[eÙU<©wlŠÖ+rQÀ8%ˆêSúƒ®Øl®B>B²w Ñhî2Äb@àAÆÍ@ЀÇid aÁZÛbŒÐÚJ }ø•_#Rý¶f®±¤•Ópg%Q³`æÀŒ¬-ìàû žÐpÕõ®œ!‹ƒ  ´ Ëðyv|]ëïÔ< J:U  0Ob¸’fpeµa¶Å¾¸‰ô¾îÙpûf™.AcX¢Ÿ·w°pÀˆRpgŸŽÇu6æbY àšÁÂýS$hkD0‚çò®*’BO)xÈ…$(ïëý¼DßQºn¢¯Ø ;4ZóòÛXJèÓ°9¹˻סX'47Ô¯¼@ ½kcy5ŽÐQo!ÈÇF²¢g€êtOö~¯w ¦ G¡XŒ!*Ñý_eàßÀªøO‡‹¾ùÆo¥Ðf—Po«²“T¿së(úéñ*:´ ‡Ø'[¬µF)õ–Öú_ýÑýѧêKö|½ôÒK/\¬Ñûe2¥ìˆÇ Üu2¼ïîR™Ò#§CM} 0@— ’+J×–eTQÈ‚ËÔTªûø °ž¯!§à:Õf}]¥{VÚWD}ô®4EÐ)¹Þ:ïªR(ÈŽÇ/ÆM—0Ø;‚h¸réN¹Õ¹à÷›Ší7é/ôÔÌE3—T¤Ù^·ÎTÌ€`Ø6p%G¥§K·šÞ IDAT­€›ÏýàËA%ÜûÏ6d+ÀÁ‹ ˜ ]ço+ ¿kÔ!$˜02°]”^õáZkX 8N"ØÍr¸ºÚT[ѵƒ ¬M%ƃ2–\¤¼ ®‚‘ƒ„ÖØʹ–¬‡Ã*¾Èƒdàž‘»gx@M”QˆzÄ xec‰65uÝ19£‰{òAQ±Qè]5ÌÈ 5›Ä.A{#Tqõ€½ îŽF¨¹yT.Uozz–w¯A±Šêç4Æ3s€ÅÙ ]iŒ+Vg>:}ƺÑõÅÍpl\Y>&R%3_2ý†4-}©Ï+¬1±Ú,þóÝÓŸ½ñÊÿüd#pÀ‘8ŽÖÚ9L>ï{uÅÔ¥‹q–R×æ:°©V8 MäÊåWr÷襭X!eúßû),—%HR»ÚÔl‘½QO¶5WeËw¼Á9 ¸ØÆh+£­}ÏÃ9+Nx¼OÚˆ³˜;gÆË9G6£6æ¬X=]îW)3µúô ›ýleïO£ôÛ•¾¨Uqµ ýU¿ÝAÔªµ­|YDõ¢0›E¿÷ÑaôÇ›è¸ó¡ŸPqŒ‘wÿüw÷w?yå•WzWš^zéåK/‘Þ°w,¨«Š6õÜÃ14Æø@¸£^# $ )¥c”Eã}óÕ S¤ÖÀ` ËÎaŸ90Âé2`¼«¾‘I)ÌôºráΧP,FïÔ ‰*Óÿ†T{ÎR@ƒ6ÏT h`î5¤ÓòG €PÞÂ/€Ž[hº ¨FYÊ2…5Ü[%š×áŒëüfòÚFlWYÒ&m.3œäMp%ün1ËŒ/ˆrü3aÆ”º{¬•†M4„ã’I‚ Išù Š×ƒgŸáî?5›@1PK8>ËQ½¯R0O*`dE4L4Á‰ª lñ+Û¥ŒERb"»³¸²zÔ©µ G›ƒ1Í‘Y*«T¤í0VVVsä,k›¢Ú'¤M9oSv·õ¿ÏE¯ùu”²’$9àjcclkËôEW‡,¬ðü®z¶•ÑòlvmGË<~mÞšFésCµ.A’kÀH—é¢Bz¬[¨ü¢T‘ËÓMüöG‡ÉÏæý¥JÕ õs—ÎÈïÞ¿ÿÏ~ÿ÷ÿÃ饗^~]ät‹+3=÷ÄsTR9»ÂYƒIy¬ièÕ1T&ØF ­é†X†²>€»Å' ,Ô–oÌ,HÍÝj„{ Q)È}î#œ±BVt ÌhëœyÙјåâÝSˆ§Ç ãLXÝ‘ñAn,P+φ!óÅÚ!K>/,v %Bå“»x;CÃ2)dP,1 øÐý€ôä¬ö÷Àlb`«kg]_«™‹Zë «„ªò *¾ÍÀœ}”ã` æ±±þ/o4°ºàgÙCVß7ÖØlÖ‹ÿróÁÿóãÿûÕWŸøY­VE’$'ºÜm̃6‹ùyX$(˜(mùÔJY•ƒ²CàH¨@ Ù .óœT©NýÂÌJÙM¦uÜ#ùÙ,ÃÂjƒŠþÅ­Í…è¼n5mÀRÛyu«ùu=΢g¶°v¶m»Ü¥Bðë¬òÎEº@ɳ™.0%·úô8ߌ`øá0J¯ŽuúB¬²gc¥¦ Ì@±F¤iQ2Y¦ŽWñÏ?9Ž±ÊU¶­ÍžP)#„ÿò£>úðƒ¼ÿgög_ÂG쥗^zi—bïà òçïÔioí“¥–SÍòÁ¸€–&à •FÊ€Ch€>t@q>ÐRÌBV.ÙiI»r 4gœðûÆ á@Š „ŠÏi¬WîÊ­f±ŽÁ¤ef› $;' ËÌ6¥Ý,<³ÞÇê…©w­!!rU3„ëò±>P°x\˜†Ë ø:ñ>D÷‡ò>â8x¦ ̲ƒ×ÛC å^5u1UBÇDõk¸Šb¤Èë()¶nYv½ Õ÷£ ˆ:Â|8€Ý4…½õyºr q÷`Ùú8f ØwÍ•kêwE Ä&ÀþtJ`DÉßi @ ¨ ø`ì#zZŸj3àëXm¤K”ÅW8\Q7‹ƒ‰pN+ú±ÅuˆÀ ‚ÌûŲ1ŠŠUÃu¹“`}tV÷wÁl€›|R{’[Qý:ÈžçW³˜!™Õ%Ä?ñׇØ&dóCi3¬m9¢ìA’…ûŸþê'ÿÍÿòý!|I$zÜñÁ˜—^zéÛðGQž‹M@ñ*‘|â ¿o;…® æ“#žkï$éoª AÎ'c_HGuo£a:RÌßÎ_•©Î8*Ü®\ ,ïŸêws—žëâˆuùÜ%­™C?ü‡eókP¶!í½œO°ý¢(ªþ·àmk×ð7Þ‡üw¿ø.…tQ¹yÀßCÊ(ŸÂw%âBK mè%{¢N ³Ì&ó †ŸäEt§0zQ>‚›(°q9 gãè8J7©º}{žüø“yônZ輫Ì'Q\<‘%¼þá‡þ›¿þ뿾տV½ôÒ˯›ì¿ùfñ—ÿàAéß<²„)ž¡ÁµD¦ñÆcpçh¡M3W—A†mí*ëp£|¡©xê;…%‹4cÀã7ôÐ 0ð¢×Þ…ež¬ê|‹‰”ú,†|=“«Ø(U<ŒÒ§HÏعþSØÿpɘ ¨JÑM°ÅÇ3{`¯¬ùl6´ ®Å½pe8±AÙì3¹£®é­ Ómû(nEg ex­,C–×`(9Úê Bp(,YLx–oe@ÔeÃ2I ‹#þ4KÜPšýñÉxÃá ¸££5œŽFpg6…Ãá° bبŸ¤©-È]Xl06< ‚±ç™'ŠÆ¡r?+ÝyU¨:Íx—zs´jt±hÀ¿ Šö€Ì 8˜¹§HÌûÞëÛ··{»ÏïüÎ9¤ ¹ˆ¹êP{9Qø(ÃE–-ÄÕ=÷ "Z,ƒÈ;Âæ™úÓDº #¢7ÈuF=þ7¿¾»Q“$ÉnÓê`²Ãx¿ÍAœû‡±Û“v‚»©­{E076³cc°ÕÜ:¶lËz‘SlŸ Ft@ ä,Œ J>ù’fÛ±dŽ(†˜èVh q}È£í ÒÚƒ  !tס»°ºsëa°R÷qÒ4Sø°!“ÊXM$4±uÞöW›Âܽ'È ±FIú|Ðl.Ü+sFŸ1`¾tèÍçþäÐwž=ä°ÜàHÇW”RúZ˜,4ÆE™ÂŠQVfµà«!šÄF bÝ,#\øEç¬ã¨|¶keñIèfÒ’@^4Ž"F^ú,XQ§ Ú¥t…%DhlÊê­‘ë+’%²V佊d|Œž„€7 tHबþ2ן0¸§Ò4À]›kk}¢ ¶Ä‹X›ï5N¯ôqE¶ïƒ.ˆ˜#gµÖ/½ýöÛoíÛ·où–èX%•TRÉ{êjÓ7M¯d-“¸®”bD¯Ü U””s$3ýËzÍÎd´<¸:¼bƒžA,î>f8戋`©ø>:%Ð`ühS¸¦ MrOêvm5/­ ^®CÜ™\‡ÚxjmˆZËYŽC—]†(ÿÔM‰ Īî¾ÏƒZ:å’¿–b0H¾“fr‚a1eÓ„‹ÐÓÿº³tQ©–Nɦ6†ƒ(Üm«˜-gX¬EÛ^ĺ^~=»¹B“¤×lÀrÃ@3®Ãx?†‰^?ý¿2â,äXß¹.’X{ëu¸4>—š-èGÊ5 ILMÝÉ6Cêu:‹t6™x”hìÌÂþuq22ã™DiÒ4¸º)w1r(è@Ö±sbȉɟQ=hAçêfè^‚¸@Ô£ÆЀ»†¬|pïÒ)QE¨È#1×=ûT¹ ¯ÆŽ˜Í@ÅLêé|ei·óÇÍAÔ¦`ö£¿xøÅ¿ø›C‡^¿\€p¹)À¥Ô¼1¦Xd3Œp-ûw5ó`µø#PŠ„Àû]lP“¤+öñi Á¬hŠ&ú‚Gö¦Ï¢’'”8† Û0 ùÞ¿ø³Õ­"¥ °ÕbAȲ£0DBõ¬$©•¢HhÙæºVШšO+ríc …â‘È6–±FBý”¿žE¥b j±GË+mƒ±ÆxXœ–µÆǹDk¼dŽ)¥v:tèä¾}ûnÅ*•TRI%ks> ˆJ5MžÃYtK3!ªWú­E: Â@¬û”†OL@®cŒÂ¦ ™cÛ&‹³æÒx%!Û‹·øÊ|_Ñ»räÍŒ£]úý: –dêÓK5WRÄ gQž&ÖŸI½ro šYÒ’HY¦Çµ:¤KêY$¶.ApZP$ÉÇuñp\DäS2o‚å©{N !ÆKqF+0Çí ]gerœ!Wœ5©â€ÊÆ u@¿1Í*ÓŽj0߬ÃÄ`ë:½ $‰}:X7ˆÐn6`f|æ XÍÑè¸Ú瀬_ò•‹,ðâ枮œb †m#Pù?P‘gÌÖa ƃxl~]ê^cW‘ðü^~{wåA ƒ0èN@{öè/´Àh_èÂîXÉÝ$ÂÇ]üÐ ,-âß”Ý#™làž¿’ý ‹Œá†nHy}æeÓìøÞ3þ½sçö8ã­&7ʼn_k½¨”Zst[(RÔ7Ê= u û;t݈. :ìÒ/ -}SŸL‚P÷ Û—l j,7†{{S ¤ó…@ Õ‚€S<+Y›L 6ؼNëf(† “au©Â_ÆŽZ ¸e}ÈÏ*·¬mecŸ k_{FÇy¬÷²voVI\i”R;Ηxàc{öì©€‘J*©¤’Ä °$Ø!%ŽÐýÔ{f ·ò¦g)ÅA„ìJPd– ~ÂÛÍ=|ºWtñD¬Ö¦¸ÝÖà~^1#uÙÏ ÖÑíñgά^O3îdˆ¤ýiA÷#è/L@ûÂй¼âî¸ ò²ŒDa¥ÀF(v‰zX™PŒ 0BͳŠ­:ù¼˜„ÎO4|„!,yÞ*Xø¼Ëº‹‡âð™@y ŒxvŠ ÔIêà ×è=L®s$àY™kÔáìÔœ™ž„ű& .¹N!ÌÃñ©i¸ÒHâ‹ø{Ú8'þ~È–m×Dl;‰;TvFc¢gÞPõ>Í´6yìãÖ_k˜gMZ$íBû$’úhËó62ÚøCЄ•™­Ð›+6'H‚¯Ú‡Xeï÷ÿ *i}ù³ì™#ke‰¬vÍ°zÖ¢hV¬‘LîÖ¸î!…?:fÔ½÷Ìœ³ÿ$Âù%­<ŒÝPæ†ú{˜kΰø!0p”€Íjno£0FÊîSö̇ê…2 hºY$FŒ1ûqç_ýÕ_]ȃ±VRI%•T’¼¿ëÑ4ƒK`ô&C,Îþœ”i=¨"ÈòCän+Zû¹Ý(ãYdîDQÉ)òÌ4„a‚Ù†bµB¢õi¢åVi“Û ]Z_«©,p£eY8›UtCÄ1QHIk1§1²+r7ª\cžxË-h¬_‚úä"`Ô dƒá)TXÜ!”-’¦’öûþuÏÝÒ³Ûÿç q0i!…Éçi4 ÄY†?|0Y˜Å°X#vq:öAJvx œ¸%À@£€Q•x+%—÷`¡Q‡•z &[ØØé@¯Q‡…©i¸Øj¦izeÖ•øGYÌjKJÔ¡uå°ã€Àqú‘è8FèÚ}á  ƒÄ aóéã~¸õMŸ‚­¸¡¤™llvQŒ`°²V.n†þRÃfë&ó$À²·t$cš6Ò±6è3mÛaVcY7nÎqe—ÑÆJHÒA¯ "kI,±© ¿Y¨ë¾up÷-ªw5¹Y²vµÖQ †@‘¥{˜›‡Œq°š’'e­4û~lÚ­( ’è/’o)‘’Œþn’Qàºÿ×sÕ‚*ßѱ1½~/÷b앵w˜¢²þ¯¦@V Ç{“šµ]©÷~jÌÀödm4&?làî»b}ê4ª·ß}>™‚²µblŒÊ’¢B™Bk}æä}FÕÊž3 dÒ:‡]♯찛Lb¥Ô¡………çüñ™ ©¤’J*áråÊÜâæ‹ŽOgE'J‘5œòk™˜ÎE”]‹ÃäÔì / àFºøë}]µßƱ¹xç[8åÒºè,E/;T¯<å;eí}4?c:€G9ë6O àÑ Ó‹ ;».e“4Ö/B4± z$“ϸP<÷RWçÎCY4ÖˆU´Å'c¯yvµqn>î S”Ù¿¶m/»§ñGnê"Dëc°SæÚ#ïa•oçFÂØ8Ô5…QAˆÒ›Ç˜¶$¼¢\pÄÊ× a®'–·¹f# ÞÚÙ°.IY㌱þ9q&W7.tŽ½+—ë[Ô%„Ž,ÈÑ ¤l)ÈüÅ,0À§üÁ ¹›YV‰aDîöå\p(xBØOˆÞ ³g¹¿< Ë7Ã`±‘–S°È×°ÉÞn¬0Iæ0/g ãëÁaø¦¨Ñ÷ü!@…i¾œ}Ü¥›aâ-ϵ»ÑÖ-½xðÉ'Ÿ¼å¸YÜj.^¼˜ «•“À¥g­®e±F$¥~”¸&CÝ{PõŽ ÝR,ý‰¬u‚ ²?€ã+î,vþ$%q]–Úµ·OÍ5^ŸëDl¼Ë\”ÊÆ­,æÈp™¹ÝpÆk"ÞóÆ¿=†°Ý~ŸÌk„Øœ€èÁ4üìOõSÜ‚ òV²n‡Åæ^#ÿ.cg z”*e÷-[ce÷¶~%ceX[×ò,ü “MëàÜÜÜÎÇüìŽ;*µJ*©¤!Ë°2ܹˆXÑ«,$G®¤XöU´i¬7y[¯’¶ÐGdTTNʾ¡ÙHc$‡ ¬Bç ЙÅYÙƒcš.8ûœ¹:@Qø­Bk´æÔ{¥ˆÖ¦<ƒÁÖâì{ÙØÄt.n„ö…m0XÚF×=àaXÛ4êf Ý&JÄÙ$v‹AâQ¢à2w壚Véæ]ÄžµÒûI—c™„’±CÐöO¤7¦kÏ­½˜âJ;Vî*Vï(à{„’[Ȩ·- Xdé6Ûö,ÄEëÍ…¹K§iØ À&9 QþðA&?^ÀçÐ;œK d…¥lƒìLlˆîd …aFŽ9cV/¥à›üI늠ß^Ë·¥L*Èõ;mHéó¨\ëì G—–¼k‹mcv“<ͶgìgŽ‰8Ð 27·îm_È;Ë05¿ ÄÒap "vÑÀ³}ÝÙ1½ôâÞÛkÉs=åüùóúá‡~>ª6QR¢hxSWCc­X ^RNåþt¡ºBõÚkB“äïV™L¾kDñØX­ÿa9Þ2Ú·ûLG_²ÑPЬG€nÿ¢Oºà“ôjöâ|íåWê‡{Qê+¦µv÷³cŠcBÏé÷…—7¹6TçZåZ¯ÿ IÒÓbã“Z}l›Ÿ¨£š–Í·[†RXo"nÙˆêÞ»lmti¥8VOÖ<;¹žå5òïÐóbh09!ÞE[}fíweÌÙfÚ |ÐÄì3½V®ûÐ57³$À"¾{üøñçwíÚur÷îÝÃ_|•TRI%·©Ìxspßÿäçàà”»‡Ù£–aG(¾ X÷bYµ „ñç0ç†ã˜:¿Vù”š9YÝÇP<{LÎ;Av# XE–°À X.÷[B{±ý²@ ŒÝ=ÛÄó\y©M™~âåqЃ±´˜Šb@å]T n)tü‰K†l³y,èÙ“œ¥i°FDBÚ¦l™8!vQôP·)YÉzB¢ø–€.«œ/èù„³’|%¶9þ˱vlNé³ NYà¿a FŒMŽC½^÷ãæ˜VÑÏãå " „Qàc ®G^f°Ø‚ÁRjS“И^Õ\v©w©BÍÄéxätK}‡;Ÿ©AEçj2Þض{c£í6É2c$ìz h@ uò¸”òmð!³ôH …Æq–|ôÊ-@½ÎK3ÖØv“ÉôÆCrà‹-献Ú)"®A¼ÈSO3pÖÏF*+¥µñ¿Óø`0Â]C­Ä„眖¬O Ϻ Åî¿Ød@w‡Òqº ›aef]¨ØÏx@ÂøQ7y†!{/¿ÊÑ[žâ@Œ´ dü{†£À͇Ÿv¦,}Ñ_~ü k0‹Ô5Ú¼tñìÏ¿úÕ?úö³pûÉMŽ(¥®Äqš;JÁ È¨Š÷°k††¡`­vݰر†ÞÅîؾñ¸yzºÖ~h¬6Ø^‹Ìz“Ž¿ó™Âé‰ö¥“¡ñ©ˆ¬5î%™=[ýNg/.Ôö]X©l÷’ÜÔÃ8é~Q¦pŽª\†Æ"$00P¿ÛÀÖûŒúø8â=ö9yÈýæD6Lo}¸ÿmî¾ðôŒÒGÎœk#¶Y6w¿Õ”y颳–à¥òºP™Q€Ù€¡¾”=Óe.<²¯e÷u¼Êîw¤ÇñÙÙÙç¿þõ¯Ÿ»7¨¤’J*¹ÕÄ€>™œ‹Ue¬É>9¯ìÛØEëª!0 2€ Â(°Ê—§«S©“ÓQò*”w9 ç=ê1Žb%M\¬óBÚ0š¾óI°aëÁ‚Ú+X"âµpŸÚ°Ü IDAT ?i¿AèÏO@¼Ü‚Úô Ô&@Õ;€‘æÀñçCÆ‘ Hv? |ho,tãGâð±Ã°›+É à‹hGHB€ ¸Ø:–1äVV©5¢]¢^æD žÔÌ/ Z†’¨KU¼ÀãϘŠ•£éQY–Qú)P"Ö‚kašX°Qxj‘¾ç +åÛ\%¿…¸€ÙßȽ){)0hÁàmvÏ‹Œ`ÐoAçÊè\3Pä@ÀJèxW!— ½› zœÝÆÇ—Ax¸U€¤ÑèßK¬?n®‰[RÊÆÑ¢~7¶§eg¹yóW.þW¿úGû õ6‘› ¹jŒI|è0‚¼+6Oo\ ãJ@„²X ô+uµ£'¾7NNÕ{ŒÕ÷(Ôë0a’ ¦;ë¿J6*ò¬»ÐCˆzÃÒÜŠ:~q¹vðâ"^J­#(v«)ƒÃǵÖ]I&ÉNŽÝcð¾{@=ÚPxhŸIéæHýÆ"Ÿ»6&Œùð}&ºk+ès³Çg \XV¸ØÉž±53 V›¿QÕê€ÀsYvíj€JYlYfTÀepi­ á¨’ZÕZÏ#â›çÎ{q÷îÝ—®ë *©¤’JnaÑ*:…F'Tñºí%å^8ÕÑyªÐ ­V›& ‰OâØ@Ï Æà:UŸAì÷`<…†`LCµ„3¯ßPÒ„ÍžR´Ó?5Q£)0bÏôop–l$ˆIRN"è]„x©ÑDê“K =ÕggÉ v̈Üž²:œë€Õw½áÐ#¼ÇŒ©À~‘ìÈ“0V$›ÃphÜé>äÎä6‰Š&‚®/b=¸EA» Ù%¶Zã/eL‹×fð°5'ðàH†·D¯8”DDÀ ÚU4h"Ì!†pãK³¿ðO¬N ” 9†ŠpÑ¢c@Æ?c‘Õ`О‚ö¥MÐ_låÁ^Á¹ÆÑçÔ'ö܇Á¯4E¯-d<Øã5‚!`­’ŸÁ€{ÖL¡{bÆÚÄçŒÁ/Ÿ=òýßßó/-®œÛGnŠ˜#‰<øàƒI\‘ûq‡‚6e±B¨ÑÞ’´\2æAÙ½äõÔ/ÖA•B#{)‚éiµÔÔfzºv ö#Ô5¥L 3)E±ë5Íf­Ü_Ñ@¿ÝW3ê{ÏÌ×,tÕBY ‡k•P,’JÖ&5D¼#IÓkðÑ»@}¬f#PžòFÖZ€Zä|w“x<6àø¾¬5ÖO£Ú¶ ñŽ1€VÍ`ؾßVz¯s*Ÿ!ÚöгJ¿“Ïߺ^Ʊ×É#tÝÓ2ô3¹B¿—cA?S0EŽ×x&’T½ˆx^z÷Ýw_yå•W®ŽpY%•TRI%¹‚Hã[l=? qEÒŽ¹;r‘5â~Š.WgÀ j™Œ=‚œ1D ^hã> q± N]4"x­ý¨ çw[Ö¦m…\ɵã“ãÕ"P€€ºH¥® ¤¿ûû“»V‡zèØÿ€ q´ êø1ÈE×æî; qî8Uèñ{nnÿˆðàÃ)ìt¡È? HÐQÔƒB›qŠ Ñj÷wã2Å„öõŒyO”ŸÀw,6…Ëò¡‰¢nÛFÙ ÊÑó5QæK­þ¹…Ú¥µ5ŽîïÛ‚Îu‚O˜¿}À¤¦!j šX‚úä"¨¨Ï2Ã@É™È7•)Ì=§Ä5ü1è= =“ÓØ.Áõ#”ÿ»NYÿ BôÿBû kc„>#ùA^Bûƒ›·m†F³éôŽàºaËYd´!ã%ñ¬B[iyû-ó„Ä/±}Æå‘cÄðasa cƒ1X¹´ºW&Sö“þ¼›Œ]ó"SÜÀÓí Ù\¸ûñw"°µ„£“ÆúÅê'ð‹}ï¸þ[¨Å À˜µní ‡^ÿÃ7öïßß+ÎÈí)7 sÄÊÜÜÜ`aaaqjjêøøøø‰(ŠÆ`£l«"éº~TvÉ(Vö[er·Æu?lð[ ~¼ ¸Žç˜+ Ýä¦áâäi®%«©8'ÆbÞJ!6[ Ö­¼{3àm@õÐ,w²±iékB².¤ÄqÌâz„žGú\¬Æ“`Fè9± –±G¤›MˆùQö9ô{Ù|…Ür¨Œò˜,Ø›ív{ç#Pcâ¿ì,¯ì[ymßîÝ»We‘ßNrÓk¢?ó3?31>>þÈø9cÌ]²ÍeWˤÌZb‘¬…M²– “£d­Õx­¿aºÑ}x¼ÞÿŒ×"„ëP«)Ð1,]^ªí9=W;ºÔ7mĈ½W³€Ë²´}eõ„®{/ÖñQ3‡ÜJÒBŒîÓzó½¦ö©–»ÑØ «!ØÞþ‚là ¹l%5Þbƒ@øÀô/ëÁÁà ¾·€¸š“÷›ÕS u­u¬…2jÙµ2±Ö8n Cäí™™™Ï?ÿüyD /J*©¤’J®I>òcoÓ¦»üù˜ˆK^×aQÔØï^irRÄ:ˆ–ùòhfew¡À OÓd¸Fˆ¥8O lëãy@9Sðóz'ÍÊÃÑ8˼`0„ÙDQ¤c©4ÔÆ»ÐX?Qk9É.D†!iƒ"¬#¶)…°éÎ; Ñl¸ ©ˆ=ø!Θl,±lqæ,š@œ "j'”<§ØÏ4ì6¡~¼Ý hÏnÞÜXFU’Íȳ¶<âÛÆ]a ×Éç A.(ñ|»&²~²54“¿Ì™žøÚ²nÿÖOüPgÿŽ;*ã››Ž9"åèÑ£ýC‡ݶmÛþF# u qÌ“Œ‘U_%Öêâb€°|ƒPÀV£Æ cšPkv²Ä{:j/ê´‰.Õ”nÖk0ÙªG¦¯ÕÙsW민{¹v¤C_©â4R«¸«3«ŽaˆéQ¦@c„„²BÁ,×ZßI`ä‘îÿD«…pgس•"¼Þºd=R Ò1#Ìæ×I(ƒÖw4Œ$uâX-}õJ¤Îöú@Öýq-RƘ(c˜HŒ|¶$ËK–‘ ’PÜP€ÖмX‘Ï}¶ÊÊ­q]'c¿~~þùO}êSç>÷¹ÏUÀH%•TRÉ ’KgµOzeï¶í|­V[÷"âúalX®¨Ê  qëÊâ#XPþIê\kYÏ๒)c>­0o ±,;K65˜XVów²,{-„8;D†f% ‚»‚2°asDè^ âöăD ¨œÝ@ÇË)ôÆÇ€PžµPT6=CºR„¬º½;,8Ã02„’oÛDëcÅ£òÊ¡uW_„2nû)WýÛ1ЯCW¿èíOʉTðŒ™Øã…fNB6vþ Sü¥g’ÓèÇŒ2DE@öžÔ±­qkÓ˜ô¦`ùÂ6èÍ»õ\ð±®jÆ÷Ù±¼ŠãïA6z í${ï¡+ˆ|4Üß¿rø(ŽŸ ¡mžé›Î¦¯îy«b%‡å§þÂ/üÂ=Q}Öó±Äåf­ÖäsdÔØ´|c$dრtÏPŒ’cƒ{6L˜É³KÍË=ÕÝg˜¬fÅ^K¼„ëɹ•e£Æ±‡?º ñc5€IÛUªôSKbà³4 peÈËë>æ Ùì%²Î6¿tãô3©CÇêÑw–XÐϵÌYR¶V«ŸaÏFè3ˆ5[ö|†˜-£ÆΗdµ¾‡ž§k\ß±1æݳgÏ~m÷îÝ'Þk%•TRI%•¼7ùÜó›ŸÐýþÿ¨þ. ®·•XæUþ¸‚(à!ƒ[ú¥öL €16 %¸x>f‚ç„2kr(æGÞ¾bZ[ÌÁ«4Ò}•«¢4¶ƒ»·ˆ½âËg± X{ãË{¢Jè¼ã¿SÔ¦W >9ªÞ)eÒ2Wã´;&ÕÑ‚´ë¥´Ú¤ Ä Âù à ·ìË5bÂø5ƒËëe])Ä%À:Ž5Œ ‡ž ‘²%0E6nÝÍV“µÙGÆä 8Oï\d/ËõIP¬<·ö Ãʹ×øø<ü³'OxÏÆ$!ë†Î)Ä 0rþè/5 ƒã˜S \1Œà{#‚,ΈȼCÖŒÝ=ûèò¥}gd¶Ü`ˆªo´ùæ©£¯ýÖ÷wýÙw†/ŠÛ[nz戔wÞygþСCoä#9cŒi(¥6­ÖjaFJ”0©œÊ˜&2.Ë5eHŠ´,Ózhý!ö€5Xê«ËW:óýSß0iE—2*;CZÃGQî‚/³€H«z™¥ýV”º1õíÚ#ÃæÌ–KÊ$±G$“C®wkX~.{æäóiD\Ÿ26YˆÝA¯§Ï«|B}—ÏSYù5Ä9:;;ûÔ /¼pjháJ*©¤’Jnˆœ8ðòù“Ûš_»»¾åMDÕ0Û±Žno–VlŸÄZ•¹nJ•^E2Ã(ov…Áš‚cƒ²+²4 ž}á÷»÷’=…Ü_ÜÀ×Ï€ RÆZŸ­òŸ·Ûï¡y?YLÚ&ïlÁX΄2O¸¢jÇ2M¼Ò„xeÀ4@Õû ç…<¾‚=Oq¨ôŒå•írÂÖ7lïwóNÖ Ë[äuð!'qEñCü=$‹Ä+ÂÊ1]†HaÍö àl¡Ä­f|jÂe«1䌙^®üu²Ñ¶Í†´•'>FHr¬e~dµÛkýí³á?ÀÈÁ»¾l“T½½• °rþ¬4Ù3Ä튚€yôa¸9_øµÌL䀃†<Ÿ¾‘ü°ƒÄ1>ȳ.€.T8Ð&þÖå™c¿ùÚÎ?þî*«ä¶—8’‹9|øðìÆ6«µZ-qµ™ðÁL(Ø@•<©œ…á2劂 Ry“.3’òOAY–ZÆC®/@, f”¹;”…”®òúPùP[èwÃÀŽ SÒÜj¢´®7WÚ@¿·%Âä¡Èª8¯@7wù=u±Ÿ-‘¶V oêô–"˜¸ð¢ Ÿ¬“lW5 8rf€ø&çi5@Ë"!놬ÌHY>›´Ny=}&è=BÏ›¼¿¬[ö…[NhÍò9Ð?cŒIÜß™ŸŸê™gž9SZ¸’J*©¤’/'NèÓï|çHt÷£Ï«ø4ªÚ=€¸ó33Û'²ON]"¼ ¯°¸“@—…X’¹I#"Œç{8`$ÅϹ‡ ÖªRn8°- J«n”F {–û›¢ þ4á÷Ipà UÎÐýÆù t\¸êð2EA01¦ É =ÆÔ³ ­¨I¿CMð´~rþ‘®#ž Ø? Br. ¬•Pñ $ÎeP³î)µǕ#)‚‰Î,æ®| AX©¥²'…(Ã@†2Є£øEËw‘n!Åñv`‚¬U”1µ±~o{Üíl\êõ°Ò·¢Ê_° ýú¡cZ¤6ÒÍ/ôåd¶ZÆ‹§(b#Z¢iCÊ9;ȵ0JPßQÝIBÏ¥GD–ãÏô qlä³!Ù\öÿ¡ø7´NqEB,³aã±Úó#q/*¥ö^¸pá™çž{îüHXI%•TRÉ —«Çöt?üCûz쥰Eõ­€8 ÆDTysê?qïzR(D<¢sÀÃíÍ.žL®ј$9ìâ­è¹ zʆӉó›:p nYVtçM¢‰2ÐĦ öÀ9×?‹0ÅȾ ¢ ŸÒß âvâÞx¦@Fq–FU*Ý€.öýÎøÊíÏŠg+¢Ð:˜I ØB¦Vy–Ë*)À:.e~¦ìWˆÞŠ©†-CÃvYQöcì¿&?çLLOdg(gd“C&6„Mç˜d\ñã觡´O–mBØ#¼ œQeÀƒ#ù¥Ùsé˜7I¦¤1è\ÞKÓ`bNÄ:eí'{÷u{¿l\ý3äÝÈì3X<z¨Ó]i#ϲgŒ´DàU×nÂÂÉXm/š//ž;¶c÷_ÿëý%Ã\‰48’ÈîÝ»ÍÑ£G—šÍæ±~¿ªÕjŵZ­eŒi–1I€€¸ò’(SÚ¤2r¥¡ÀÉ°X&R‘£åCJšT­Ð¶ M†Ièùb1ýi¨\%éK²>Þén¯k½!6»IŠ¡~:ƒÛ$T~PòY´FÇÖøß0R w¨ Üæ¸âöãAcÄF•'/^³¬àÂ\¤Îô´Žå}qw(ùÝ0pÀþ t ÄdŠ¸!Ì-¾é†‚§Ò6È€­´/¡ƒt…ÖG Ì„Æ&Ÿ–Y¥Ô«.\øæ /¼p9XQ%•TRI%?09‘°HŽ¾q±ß꼪{ƒï¶Zë“ øÓf k~“• 0áÃ3s EœnÄž xS¾œrj•-¢<–2 {´a¥¹SE±|–º”(Ò‚Aë0¬Èúmٙ……€4¥ÜÁˆ1ýâöè~+UaÒ|¨iÂ>R'ÿ˜îßÈ¡Ô1tÒàG>Ž|¾9KÀ8€ÀŽPàŒ2Pö³xþòç’!ç/|Ô*Êô^²LdCA¼\’óÎøÔxAçpàP¡ù<«bîXí‰3^IyŒ›âããaCÒ|Çì¡Œ Œ(t¦ }éè^0Qh µ¢øÛ?Â>½ppVˆ šZÄóNh1ÄC¬O$Oš™ÈÓœþf’vpTâwöäžß{uçÿSű[ƒ|àÁ+³³³ñ™3g®´Z­³³³ç6oÞœÆkƘ‰(ŠPº¹„¬ÅVä')û ÃÊ‘"™%¡öÈ{„”? /íQ×÷ƒ*~c¤\@4Ñíooèxƒ…µ“\ô=­aeЇ•Áb§/µZ ƒÈ&›ý®Rp$£Õqš!í$69[$Ÿ3v¬"³xa®ÕˆºÌ­kÔùV&>Ú¿%ø ÿ–R®È¶†Aˆ‡±E€¼d{‡C%ãŒïév»ýÂÂÂÂkÏ?ÿür¨P%•TRI%7‡,ÏÌô/>|2Æø; Sõ·×ES]@¨#¨M™nƒÅ—äß±Áø2DvÍÕ"#1“$.H-Ì+Y¤5D$À‡ÕÏr5—çî;ìOáÚÔmèéeÊ¿½Öª€©å¿W݃¸×G)8¢"CÆŽ(È® ,ô/bÀàE]Dhìyv( ò+ïmÇH„6!ý# w‰È™K¾‰pdz¢Ä­FÖì@; U(žÞGwv߃]2@†¦$¬ ’ªÝYͶ//¡µ‚þò4´g7Cq Œö  ×®¯}ØÓŃ5–°®‚ÃΟú´òukµ÷í% &ŠÅ£Ìï{çÕ/í}á/®„î^I¹Ü2àˆ•™™™þììì¥^¯wúôéÓ'îºë®cL1O*¥*NRy¢Vh©…¬ØÃ$Z”Y¾C. ÃÑÛÀj×H‘Šž”ƒHÞV™gÖ*IÚþ‰^/GòÁ´9Ñ“UÄûX‰°Ò琉ŽÔ¢(§åÑ@S™­QY°,šÊ×m솚Þ'/§0rF7$°=ô²Âssçãp‚–“µÉ°ø"´ŸÒ M2¾Bl®2Ò F®yZ†¾(È"ßø(û¾L’À«Æ˜CJ©çfffìÚµ«­ë¬’J*©¤’÷G/mÏ¿»ï¨þõßØsiSôÚ¦húŠA˜À;0¯ÜH«÷PüÂ3 ¬²HuZj¹¶åÝÞP*ÝU–zâ\M(h`ÿK¨)¶ŒÕIàÇ´]!íÛkŠûl~ð€ÿä»ÄU¾ Z纑‰ƒH¶í|Mû~¡¨ÆÈbdŒÑŸcý54[² ¼º¸:³› n7²>b}Û²rÔ¨Ó—kˆ‹ÂŸÏRL™J¾"æ:S'ßx‹ëaÏø¶ÑÌX~<]»æëQ¬þÕÛ§ž}æð®§ÚPÉš¥v‹™9vìØ<Ì÷ûýóýèGß\·nÝc>™fW-I*Ó†KZV‡§£ì·QË„”²4À¡ºG1dŸWKZ#Ãź ¶Iðj’æ«“²IbXèõ`ºÑ€u­1˃¬¢Ø`Õ¹‡±µÃÅ<ŽÂÁ˜4M2GìÄHzÇ‚îü+ öoví ®("CtAŠÁÕKãyðçÆ3xò³q\ƒÎü&h_Z¦WóM1‰Ï—…Üóãà;ŒôË­Ùm/Ä‚Ÿ * ‘Œ$»š|Ìü½Üºgž÷Ž+b×óTÇt¾Ð}÷ÙýGvïv¡+Y›Üªàˆ“™™™å .¬<øàƒç}ôшøicÌÇ`J™Qy}H¤‚RèÊ,Ý!à¥Lñ+k›TnGé_H¼ž9o1QeÄ= ^ (À¿`ð¢«p¹Ã|¯ ëM˜n4a¬Ñà~«vPfÑ cŸ½\»ƒ8]ü«Ô¿ µ@áÓ8ó&¹`È:“ëf5)\Êž )ôÚa Š¬cØú,[ïe¿­6.²l.}­õÞK—.=ÿ /\À4’\%•TRI%XA4 W~îŸN¿þÅ/>ùößþ;;±†ÿ þ]Üi±& ÄÞRb€V™FC4IÂæp ›UŠÓ`3ÅIyFtª@YÃ怈Éó£Ú3ˆSê’ØdV1L®Íã5 Ó¦CnYPX«ÏegËÖ$ÝšLÀ»aw0å™ï1ÍÞ gI‚\bÂ$é$.7S0X‡úô2Ô§–AÕWX*bö/xPV_Æ—¿!™ìüåãfÀ"ë!Ùi ÁãhÐûQ˜'°ÉiW°™òŸZt]€Åù0à×-æìÃà2^)ôg]?‰¶,Uö)Âø€¥,†øÈ8"û¦7ÀƒṽfЂöå-н2‘¦‡æϯŒ~(eÌ;?d|™‹Kay’5¤½ûQÞg jM˜Üö‘·€V€ö . +æ+ݸý…ÏÜÕ9¸ãvWÆ·k!SykÊ<Ð|ä‘GQJ}V)uŸÖº¶8¥ÎJˆe"¿¦èÂV)c§ÈßËd %ÔÏQþ^k}·“ÔµÛ´¸ô“½þƒ†dPr‚,‹½)›/T°¾Ñ€ “¤^g¾…̹Qg‡¨4® /’XAÈvK]jì 5@ï¢Â=§êµ½]­£²+®‡Œ\ÈÏ¡2ÞEZVöiØýC`NÙx Š`dÿþýûŸÝ»wïÙ2ˆ•TRI%•ü@Ń[}tâ¡î¿Œ~U¡ú”hzýÊ+ˆN‘öZšsY° †g˜‚Bl•²ô—´¼ÉSýR ;O…Ÿ)h˜,æºpApÄÆ%±×R5ÔŠr GAØÇ» °ô¸Ô-…2¨ß€HJoxØýÖù ªÙ‡úôÔ§æ£~Ö.’G9S,)›ƒ‹è9 ;—ðä‹¢û„e’bl’3™€ƒ6n’è½…Ke|8EÛ34è<»€(ì255•øÿoÊÓÿ²X’ªIÆ$²,–ˆ‡O‘¬6´¬ý,ã1P%KÆYPC²fØ“ôšQâˆÐ ¡˜"2fðúnw6IdL}¢?øP#Ž7ys’óB‘rk_Œö¥˜Ä$™ëvÒ±¬!¦1I ¾ªJ‘³‡I³ÑÄ"X®ó›dbmÊ [/!ž_¬E3ƒ,.FpÊ9§ë,#$Zd«Y%Fko(^‰|žŒH‡zèf‡%™xd°Vz¯Ðsz&µÖ±1æsçÎí|íµ×*`¤’J*©ä•Ïþó°2;Û;}èÕý­éM/NLm÷'1ﬦOÏ”ö0àö0b±GØ]œÞ•±E3`$¶†ûÝEFæ*“}T¾<Ë>‰WƳÎ(æºà7pϼ8Ë6Ãb(‹\y¿—‡^ 'iè \‘¤Öíqˆ»cIÄVˆ½T eóP"x!nRá†û.­0QŽ‘é¹Ðƒ )cHíºû›U‚ FÈ83ƈ蛭Îê⪆06á³Õ¸¶ hå«&é!¡pÃÓ,³k=íÄÝh a1Å|àÝš±,­âÞ¬ÌÜ ½+cY3XPزÕ⇒ž—Q¸¢û³¢­“2pÀR¨|Û,èåê}Š|W‰“vèZ ©³ öÁà7ÎxsÇ›/ýåÞá=¬dT¹íÀ+gÏžm:tèÝûî»ïd­V‹”R[2}ÖûR…ÐÓýÃb˘’`«V( "•0y?ª|ÑߨâäA) hy©8†€©ìIß¹¬”„® ¹U$GÆ{½{8Â6Æ¥ßY95B¸%)H’Ä$I,?IàÖ(Ï9Ÿ¼õU-JKb ÝAŒØ—sá€ASÓµ0XŠÔÙ903q挬U *È5DתÊæÙ–‹ã8XV‰”ÛòÙS"(+}N À¡íòì„Á¹~é}é:–k<ð,'þžï^½zõé]»v¹-}%•TRI%pùì;s'ä®on5­ïÖL½† À:æ„ߢ™Ó ,Ýž¯$‚ÖA–õ¬¯dYýËíW*"1DSH½.Oj³m±-TTñ³® H!¹5rÇ^œ)Äâ| 4 7R:埔wýbgß¼a’x°Æ€é× ¿4¦?)PQÏGídMçmµ¢HB8pðßPâÎ[2Í/™«üaL°Ø!2Ò(UÔ‹g. k+¸@þlÚÓ¥ÁÄÔ¤/Ëmïèlòù²É\2ÏT)» Ûܧfnò‡JYì@GÐo¯ƒö…mÐ[hÐÑ?g XòŸ3‡À `°ÕÏ|›€2Zì¼Ö²ç˜ŸÐ-â¿öàKVܸëòµÐE€ÝW.ù—ß~æß½Y˜üJ޳ܶàH.æèÑ£W>ö±Œãø"N%ÿ7‰gºNéx)+Ĉ”¾TÑ ’"Æ!2ìïô! ªIŮ̲-%„ —• Éjn «9·ªÔp¤Û»·kÏ¡À2»Œ{¦˜p¾I¤ïXg1°Ôï¥Ùm’µ<^G²y t ½±¹Û`1;N™ü7¿Œ5@¼„pv±]4y< ܬšÉïåz¥"Ë„ê’Ì,ù»dH€Cw@@éþ2ŒABAKú}ö¬H`Dk}d~~þ©;wVŒ‘J*©¤’ÛM0ç½~ò#ÿÙgžït{o!¨-ˆ©!®æu(où¦î²TArڜ˚’¹Ï¤À„»ÖÉcðg<˜Üõ&b¦ ±L;<“*ôtèÐ…íÛ· ³µZm: ØjA’2`ÄŠ´¤ƒ°"SÀ#§€ºÕP«¸´TSèÃR¢Œ†Ä²,e£ì>¡ßhÛo'IÝjz½{ë1lL·G¤äsïÞ‚`Øgû"N$i÷ûÙæ£0M œx¶Ö/uÌ_ÎÈ蟖Jˆƒ„Ós 8’&Ñá –$äú†’˜$es¾ƒHº„ÑCK¬¤ÏW“m¥÷ñÜÒuKÛBÁNÏ=í¯1fEk½¯ßï?õ•¯|åBi'+©¤’J*¹ååè¾Wû§½vp붇žÓºw$ª7·!ª€éœi³^G"i8˜Bþ?6Žˆ òšïè¡3„ƒœåºo( Ê\ÒˆÜ"oªG{7ãÿËÎ5qû¬ñ¿r;>âû)»6p¦ÈX"D;µâ@ßoн:Ä+“ MP‘Å|<H")¸d(€îLGoo\0N}fô¸HÎô ä]uÈùNŒ«‡°'¤˜Â¼ÐCˆêŒMŒ97ŸôldLà~ä^ɱTaØrå8ðDX1¼¹èCþ0:‚îüVX™Ùº¹uG…ÈW+é%{>5b׉v}–1o\½ÞÊ3Aè8Ø6W7æ-ÏŸ±$·äUÃSí¥K¿ñÂ_üΤVrÍr{i«#Êg>ó™ãã㟘žžþ„ÖzKEõ² ©«IYðÖ2Y-ûGY`XY~µzÊê¥?UVšÑ$ Ⱥyaù§&z½ûSŽ ¾–Á}gƒÚgïNbE¢J»»0s½iÖj°uýzî*cËçaltº/€Î™H}çLMЈnñ¬„T–¹^k£,øêjå†WTY–¥}“ßAà9Ìšd(çµÖ¯ÏÍͽøÌ3Ï,\ÃTRI%•Tr‹I´õ±Ÿýïî«¢¿?¾~Ë/#˜‡°•|Ï}(„A~a® ÖÐîã-õ±[ C*oLù§‚¡e¯ÏÆÈ°Å@«$[+:)ïÔPÚÇbÏÈ)€â¬CØ®Ò] ¢WÄáž¹,¹¡*â‰á£nÒÌ6µÉP. ÆAw(“M¥ç2Ž}P SŒcBÆv¯Ôeˆ¤þ-›KÁ¾ãÆËfS¢µ¶›ÿQoÔaã¶ÍùÇâuÜ™†º/1—*Ñ6Pvèzakž²ƒl°V^«í«´ ;·Ú—¦5¶>Å$¸g‚Ž­›jÚ~—‚žQeĺt”Ã3βþXf}|-s,ëŒe?åóARvû¡I#—œ ÿiþÌÙßß½ów+ãÛ ’Š93gδ?ùÉOž<{öìé(Šâz½žl³À:FÄjñèç²À–´Þa, IãY´G©§¬ÎQÊÞNqC®E"€Úd·w_CÇ\0k‹dSÐ#:ì®î(v¹0@D¼ŠëQ“­VÆ¡J'$×Q7›äßTõO^Ĉý…g^ŠóŒ¾Òelµ5Yæâõ^$t²R‡„"ü’í!Y ²ýÔ}.ôŒÑÿÓïÈs–Tš¤èýf¿ßù©§žZ¾á ¬’J*©¤’”$A[gŽî›Û8Þÿî²jîÁF£_Çú4 ¬ÏŽ Ä/Ú vO”gï"C$ýÃf•ñgk%w™‚Šk‡ ed׉³ MÉJ!Â0…Æ[k|X½,ZîùoÖ’ïëô ,^bµäåbÝmBÜ£kY{jÉ}5rÏèðñ7æ“ë¶e'ŸàyŸ¦Êlb«¼»{X”¹•É¹JÖ‘˜SÐ_Q|‘g«Ÿœ`:…±€YªÀÀŒNrv}X” ¯Ç¯9NôwÒ°üsÜoA÷òfè\ž3(ª´(.M­,náSŒ(F ßRÆ'2…ƒÞz®J¡I¢}i¢­~ïܱýÿ÷·ŸûýË…‹*¹nRi¸«ÈÖ­['Ö¯_÷ý÷ßÿñV«õ°1f³R§m-ó²27:Mjèþ²-äÆHÂÙº°øÄX°Ý¥òuwa¬ Íg^¤ãÙ3!Jrõx­[¦×‰ù³/pB ”Ö'tØ|^VNÕ£oŸÔ¡Xk×°QØK¡Ô¹ï×z…aR–ÚwXj`+elr&Ù}Žu»Ýož8qâàž={úׯ‡•TRI%•Ü’b ÞõÑOmll½ëG·n¸÷ï5 ö_€R÷çy>¸\°BWsõ~_ЭîÛàâAYÄÇþ0ùgÇJ LBJ˜ AOKh¡ÀQ@a•À“œñbÁCÝD”lÄP_†h¬ Q« ¨…1-ÖZÒ' âzÃâí‘1—°[œµå˜Q,®cMZU|hüÚÒE<“*ž¦BžëÂ<t¬_ôzÿ¦}âà³»wÿI*¹¡R«†w¸ÌÌÌ,ÏÌÌ€swß}÷[SSSj­UJm]X˜\o×XÅ@ÊZÚ²–öUÀJ¹ä[„gî ßmÎàA nNi–Òý¸àOœŸÚˆ|ðUK¥u²Wo QÊ M¹d!@`”ù¾‘ëAœ‘åBî0£Ôm¯-»žÔ#âá+W®ôÅ/~ñÔ'?ùÉ·›Í棈ø˜ÖzÚÖ@' E !÷À~/­ö£Æg(“k‰7R#å’d}1$Ç=6ŠíÒ›}‰ò€©L¡GüÀÃC¢yc@º_:Ô›íjì°tCÈ¿·ëIÃX«­ÁÕØ&£Hëc»cXlú½íoÙµ%}ê!âþ7ß|óëo½õÖtΩ•TRI%•T2¢ š€‹ŸýчŸÿÖ·ÞøÞ‡?õ‹_‹PýW€ø_c¶+ñ§ “YM,Ô.!©Ó)¤‘e±¡J&‡ <СÀÇ,C¦YD)8×^@‚®:v†cÂê<® WÄ‹n4ÊÇ!åÃφD)ÏFºè”9buÔ¤¼†4³îOÀ`y ¢VjS+)£UܶÈð³žk vÑóŠ¾ûｫ‡±¤^8PF‰AÁÆ—s Áac)†xR 7Ðx#n®rC[— ÷µkÍx¦ iÇ›ˆ›3Z`¨ÝùMй¼âN4†,¤cÄ?|‚ æHßÏyΠȲbèÐYªŠ +†Í¹Î Ó'>ž~(̲AóLçrçÿ\ßóÙÊøö¾É(¬°J„ìرCýñÿqó±Ç»³^¯ÿ-xD)55,øã¨Êàjî/¡ë‡]3ÊïÃÚPÉ{—ĭ接¥Ÿœèô0ÊÇCÅ”Ù!-=…ƒyo{#ñéÀ`¢^‡ÍÓë ²ñ2Ħ,ƒ‘ü-oCqñT]½z!ŠŽØíuT`á½®1YýûZÀ¾Õê=—£”§mWJõµÖû<øÌo¼qîš[I%•TRI%¹<ñĵ½{÷N<ü¹ÿþѺ©ý3õwa+OÝêÉ$):gxq%.·iÞÂjõÁ)¹òÉ,ù@ƒ°rŽ’F"R¤/qìµVwX¶” S–Ywö@ 0jï΃Éò¶©—³Lx[}‹‘öY¶G(îÎ §CcÝ2Ô§@ÕÚþ®$í/U”Ýèpü cÄÐ$†,Ó ¹¯5/*yÛ@VÙuéü©ßxåéßÛX!•¼Ri¿×INœ81W¯×_¨×ëfŒÙ§µ„j–Á$åoÔKª †®•Ëø$ÃbG€PHGUp+)ÇàÈAît´Dª÷tÛUüu 6]šç.æ¸OA¦Ø!Ⱦ´û/€c(›DåÿÇ|“Âü ¢ߪC1m†"òïÕÖè0¡÷µŸË€Á÷²Cí£ñSèg+ˆ8H€‘^¯W#•TRI%•¼?‚hÎ=ÿ§¯¼{üÿêõûÿ |%Q¢ì½ñ*¢[æ©%Tqå1Í€ÄO ÝòHé6½«L»³ 0 <Ád”Uê i§Í€¤\é¤J»Ï,ã­ÿ” à2î€ÙÎ\°@²k}Ù û‚°9Òσz—×ÁÊÙmÐ›Û zÐpem¯ e¥B2$À*¬×Î#°ô¿˜Æ¬±lôH€=c’¹Ï¦ÝÏ%Šqw°†²‡[„#¹Î_#L ë¤ÀH¿=Kç·Bw¾éÖ«lPµ çW˜ô¬^M&„Œ óÎwNn¼ã‡¾YkNV ïBÀ͉™Ú'L¥Š!AmøNƵæ JXðÁž\/6O³àç ú É~ïYù&õB·±8p‘ƒ'D‡/°kY ÑðîÎÊJˆ­“Ç8vÞçž#ÜšA´)ViÐÖ¸3 ˆ PQPi³1CdKó€¤D''£Vd1°¶åm§ŸY·“ö*W}„‘Í3rÄ°åuÖuhƒŠòYUà,}n­KÝ8ËØ)/!Ÿ€âÖi„VÐ_Y+ç·À`¹AÖØ[œßàÛ·‹²\(è‚Ð.ûÚ³BŒè§ÛorÊugúÌ-Š.pcd+4Æ,šgºƒÎ¿øú—þ·Ã[Tò>JŽÜ9qâDïСCgï¾ûîýZëùz½>¡”3ÆDé»E!T ¤@ •á0bÔïlY{ ÜÈû•Õ;LnW`$•€#ýî½õXo¤{U×#¤ß»o즄jšI#Š`¼Ùb ä>¶J±¿i]týRðÐ~O?Ó~Ñr´¬ÖzYkýí .<½sçÎ¹Ò ¯¤’J*©¤’,NX9}ø;{×?ðÀNÝ‹ÏÖêÍM ÕúD—•vs{~`J^”dÿ#jeea¯Ú}Ÿ_¬’/ÕGbØàp™³C IDAT…Õ½ÁždIì‰ÒNdÙÈzç!!y?©èºûžbßÓû°sIR^èáöžº¯ ^iÁ =‘ÙŒb ’(åX¾µèAr0t®3 €ò¿¡ŸH7v˜ƒ!B“·cFýC, •3<¼¾îëµí‰Rp¤Å™E˜õdzC€œ?=Øåê´ë,›È§ÄšÉÖY úËëaåâwnÌ<À…þÖò´%tãB]Ø)âæÖ“á+ ü‚t¡ä04¾ùs#s<ñvû¶9Ž7FÆèø2€ùûWfÿ峺ã Tò— ¹rìرî‘#GNlß¾ýݹ¹¹öÄÄD$¼»æO“TCt+eŒÕD–+SJ%ƒåv6®‡$Ì‘ñN{S› v·GþFõŒ±?:¥É…EÏ®mÖj0Ñlºïp«€[oD"q+eSêèÛW_XDÂJrßÄJ(fH@g×vY]!I®‰¢ˆe÷fý”"Ìë3ˆx_¼zõê®]»vµƒ7©¤’J*©¤’÷YfÞÙ»xîÈž×6Þqß‹Ý:^©cc="Œ'q4­Úh *^µgû+8`ßAø¹3KÁJ‚–:Ú:•Ð›_k€c àµfÛ&’åÆ)ºè@ /^!¦g${½;³J—ÚkGf¡¨ˆ?“p{ŸüdAqda\²ÄÊ` VZ »ãY?£5 ÏBÏ«0Œýa ?ºÙ¿H PÔ]ƒ_oYBèêñŒ›y'O L2øÔSpdÌßÅ<ÿž,æ邱‚E6’CSƒÞÂhÏn„¸Ý ˇŸŸY<˜R‚0Y“@ÐAè–NBê²Ä}ËíYÙ7Ñ3L€¤wŽ·W ãîÇDz_òÕ–œ:OÄ~inþww=ùÛ•ñí&‘ yäÈ‘#+§N:¾yóæÓµZm9Š¢qDl!¢ÿQ˜"ÒåeTÅrTöG| ±G*M@mªÛ»¯6ð€~`äÉÿù&…äMçÁºtZ·+!Wà·i‘ßífhýE5ÀÊU_FX¢@Iì¿ÃÜbBkW®³µ‚o¶|ÈM¦¬žkȳEÿÄ,I¾8¿¸¸ø,¼þÔSOõFnl%•TRI%•¼Oráä[W.<¶ýÕé33{tkÝ¥†ªm@e¦MžÙ&ßèè1ƒHõ*,TrPE[úƒ ±¤{&„:Àð¿QqåÚ‚!¶@Èö}åýæŠW²=d 7FQ–€4"ñù~Ó.‡ÎÈIÇËéà9àÓO@’1Ѓ±¬? ‹Då ;S–‘°åôظ¥¯ÜgÚ# u¡ûÎöÅDz¥cXo6 51ÆArb5x#ø gý†ÙlÒ ‚‰›ÐÛ˜#º[weyè…v‘¸ªÉ”±Sd@×âœ#PÐÜó`ŒwÁöëNžu X’‡BJ¹"&‰Kù–Öƒuúäÿñµ§ÿírI³*ùHŽ¼bN:5ßl6O­¬¬œžžî(¥Æ1ãX¢Ý…Ü ®…ѱÖø¡X&•Œ.YÌ‘Þö†Öœa… ?'ßÉHÖ¡‹œ €±zZ†³ú`˜8âsiõÈÿ ®ÌEx¼EË4øªIè¿!0M²MB2J<›Œo„õ-À‘‡2Ú$W1Ç`dqqqÿÎ;ûkºq%•TRI%•¼Ÿrà€¹xöð¹h3~W/®ì›Ú¼ˆ€acNûÈ78Þ&,|IO&Ò^DQ÷ÄâO°ŠaŽÝ—ë®…r.c‹0j ¸—sU"ŽÁ`<$Gƒ}&0J~a½  èk¦lkA%òµÒÃ.;üÁ»ì<Ð5&zW²~‰–æÿë.ñ÷õì(肆WºØûí“W_ÿêÛOÿy*¹©¤GÞg™Ï;7÷k¿ök§Z­Öù±±±$âøú„IB[BƒDZ)SHqHU*kU*ÆȵI”ÆéßÛˆÍC÷‡|X}i曕)¼Â¾Î÷‚$ÍZ­c98ℲmY²!ºMPJKW£èø2˜ gkPF Í‚DÝnÜF<)ûíF¯7 îÐl8„Í’!ï:tè¹-[¶Ù¹sg|CUI%•TRI%×IÎœüê?û‡g¾¹óéïbcìíæØÔ’¼ ¦ŒpoðªSç½ÒªŠ©dÙ‰„Òm‚ ²Ó ópAfArÎ ®ì7tŠ=‹"\L \—¢‰¹™ 9{ X”Ÿy¨g U®MŒ$é¶Àè:… ’¸$ä´æB~Ð $M-í-eÙä„ înÙ†Om89<ÖZ4æË0á`‚m戴=#0rÍXGô–7@gvô&ÒqÙS’å ˜YƶÛ'ò^ƒÓN¥`„æS‹¤>¤?’¸0ô 'ºujW£Í’¸gãؾ¿Ð=ýê‹{wíªŒo7¡T´€€`’–íܹ•ŸþéŸ>üÊ+¯ì\YYùscÌw1Ëz?“aéGW+;ªT)z¯˜$Ôu’ t–6ùV^–û¾ ` Û¤•Òm¬Íœ¸Ñh²!'±P¤fPI(=4]w¶L(M#bk„äZÓõ†úº·m³1f µ>tüøñg~ññØ“O>Y#•TRI%•| dÇŽúÔ©·®nˆÎ½ðÆëÿßouÛíÿÁ€ù3úJIX>Ö„p›qÇã/û„íè†þ pgç Câ§B‚^ôADmºTjü±Ê®a§"S¸RÃ)Kü‰ÑŠ¶ÉöÆ~o$—ÎUt•eYIâ‘,· wet/lî•;`Ðc8 Áùa˶‡Øëع‘|.Œ¡ ÊZÉ¥<ë ë]VRï!…œá«%càc¸¹åå"Ód%©;s[`ùüfè/Öèo ÞÊ÷ÍM|Þ1ãÚÍÒî¢JSûÒ,J|»‰akÅž›Á1”•b<ðCü†¦26y@VcϘv-dc¡ºFë]'ö¿üùÜ]{yÏž=0r“Ê5ÕV2’ìرCýáþáا?ýéû1ŸŽ¢è‡“x$ÃMR)+G¿_­.ª0ŽzM%Ã¥¡õØ–Å¥ŸœìõHˆ öí6†üjû;ÜælˆõÀí0ÝjÁÆÉIgQ±BÝj )OH¹:"¬œ;Ù¨½xEáå 2l-¬uí¬e]®UF©¯œé+¥Ÿ:ujç‹/¾x&0¯[£*©¤’J*©ä$O<ñDmL­ÿøÔÃ?‘úÇúçUSƤp”zx¯Ü»t¿y7Ø©Å ­RY'Š'Ut“HO« ;Ú1ˆ¢k|\3Û¨c5½¿!Š6wÇa1Ç ÎÒ —ºFð]VT¡ãœYÁÇ0õ)¬ÅPŸZúô<¨z—̃g®¬Y³=‚3Šyðˆ‚[.%±ì'‰Uâ¾%,’±é X·a]–ÙFN7@A‹”ãå™+†]˜¸u®nö¥É4moùø{Fÿ^§ìkCbü>¯*@q¿ ú AêÔZŒ•äé䪷& ¿¦ó Ø8†´Ï)ù†Àœ5âãë1–uL,”J’Ÿs)Ãß\A܇•ÙmО£éMû¹´Ì:dŽÓpöžF…àý~ÒºDׂã€Í+’yðì³Å.#Æ2Ž÷ÝÀçÎû_÷|ã?|¿:cÞüR1GnB1Æà/ýÒ/MÀ§•R?©”Ú [9ŠE?T–JÈz¿–zG•Û•’2G––|²Ó}(1-è|SJgó4j°Ð!w–jÈ¡ñ?`Ck ÖML¤)z b ùµIýÊÕÍ€v³¬ðô±zôâU€y[>VKÀE+={v ‹—“ P¾d}=†'äÞ.¥±½ÃÖ18¦ƒh¾Ök_úg¾ôÛG K®’›RjÕ´Ü|’£Š‹O<ñÄ77mÚtTký“ð¨ Ú*•Äa (±ÔËßeÙë fT®9XØ(t¾ËÜàAÈ‹^Óà"îîög¸1ìwº[À[Iòá}Z‚¨dr c†@L“®8!vùŒYB%F†7e’#Ƙ½N篟~úéù‘.ª¤’J*©¤’°ägÌ™xàßÜùÈÏ¿l¢ú¯àÏ£1SR`G{$g¹Q9‚(‘Ž9áÙVm÷J{rU¼‡uóHAå”OÆ~5´ ù䊱?ì=Ñ‘Ê3ße}E™˜Å¯°g/´AI-¸`:ïp¦I– –XM¿V,VIRß`© º}D“ÐØ0QcÙ54xu߀ŒYƒ6n 9Ø@ÏŠvF4E4DaBA¦óKÇÐÆFexß bšü3èŽCçâè-Žå‡q,šµÇ»GöŽ1üb1kìí³#eÀ‡0ŒïƒQüš` ÛI[)Ãæ(³…<Ž5¥Ûñ¯Û½î¿øú—~û\É¢«ä&”*[ÍM,'NœÐ¼ºaÆwZ­Ö¬Rj§Œ1Êf¨¡™6†Éjeh¶›ÕR°®5%ðí*i¶šNïÞ†Ö›ìAöq·ÑøÔdÒî@ƒjýÿí½ t×yçù}·ê-Ø €›HJ¢(Ù’Ej¥eK–lËq;¶Ú±b§G>‰gº;N§Óçx:=Ó>“ÎätŸ mOÒ‰ÛÓvÚJ{k9J;m3Ž"Å%kƒ%Š«H‘¢n’ @b_ð¶ºs꽺÷~÷¾z (qÁòýl À{µ¿êÖ¿þßÿ£ÂHiH"’‰D©[úÜÔ2¬@ VÔ˜B‚³~)!/plØ󺦥ÌÑ?ðîy£Dš¤N_›¥ ŒQád¶öÔn?ûjÓ‹Eð2~³s1ˆKÄÚ:&«lLýåcd“PÈÔA¯LäEÁjÒ"£%Rg±(Ôñ"z€^3—…}À*M2•„d*UþTy@çŒa‘n˜³¨ð )f Ó¿ ò¡0€)K‰)µBz8î µrI7YYEÙ–ry˜ÎT1!ÛXc'HÄ,2¸&Î$)å„ü§þÓ¶üѹÊ#ËÌgXYôôôŽ?Þ[WWw"Nç=Ï E? mU¿ÁT° íU³ suŠÄ‰',˜T'Gr¹ëÅ Í5Dãþ9¦×&zÁÐ×:÷ƒu‰$Ô$æ5¡±®SjðA_rÖ•ñ°{ §ZÝÆýS¸Î%zT£9ÐsÈ Ú¢ç-u—Ðó+î4rö¶k_i˜Ú$k†<Ë°Q&ËCsÖ7Œúé987þ$ïBÝHGîS]Ï…zï–Ý`$:Пü£v ‚éfCs9è8©âfZ -äF:Æ­ªå«ÌˆŽ¿q“, ²_@ºî˜írÛ [O°JÁ¥ÁtŠ3µAÐ E’€„غڔÔu˜”0b&’DUPÐÔË+OÖ¦!UrŽ ž 8Ô½COYPó¡0ÕÓ+J¥BR¢ÞN´v­)wÙ42RVG£Ï/ë $ɘ±ç®8'ô6Ź`¬ó†ŽÚ©ÀRz¯ˆ]Yøó§ú¾ûÜ?üé$0 G ]]]cwÞyçÉÓ§Ow×ÔÔä<ÏRÊZ*’(ܲ„8W‰;í\aA¤:á±ñüúlv}2šÃC®KjÀ„¤BÌq´ÿXÇcu¡²$Œ¤I0%Ôì?楽G+ÐëQ—€âè'º Bä©H¦„ê¾p$Õ¶Ÿº=âÎ9*~Ðl×…âŠv³‰x4µÞ¡½}}}Ïž>}ú;väæò92 Ã0Ì’¡½]ž?}èì5 ·¿66vrwMcë”@ô°YB(’àOâDPÎI]å±X%UÆ $¼Uv…”ZQEÔ²•ã Ñã×£þã!îvÄXUª8l±C’U£³x$Ù(@òèLYèc©nì‘M² ˜IAqº¶$>%¡Ä¾á#fè}%ŠÙWA‘)¾Q/¥jR¥²šhôg—©é‰£¤<¿€b! ù‰È ´Ba:¡7o–C«³btY3>6 šÑ.m"n3±¬X®q‘󅜧$bUÖTpÒ%dâLÉR*d r~ðøO·oû6»’0,Ž,PÚÛÛeooïx2™ì>ÕÔÔ4#„HJ)ëPùûb¨Ö3œ¹t¨‹_)s$›»!Y š‰˜®ÿ¸+èi*ŒTs‘è0/D¨K&!éûOA$ššSëêAD³¼’sdtØÝyÄ<-§¡ÛèŠT؈;ŸhžH\IL\¦HܹéÎ[MÜ«F¼ŠˆÝ===¿˜œœ<|ðàÁü%ü¸†afQÑ×·/ì;Ñ·²~ùÎ{ëZÖŒ À4¢h• ËÍÐܨÒÒ Ðc*BÒ^RzCôg#l(õÀv¦TŽܱÍ,cZj`Ý ëÿ:c•87¬[ž ®â,ÞGaüº"‰6”X9(¨E’ [[jý[j³,Š 0fùz{Üㆶ B¶A9LÒuiH¤ö‚èü–S£ü¥˜­‡™áV˜j*µ)6’Q¼(b+â+1¥541¥r]ÖÒ:èÒ:žÖÙ.ߢ£í ÷‰úlÑv´8Sn’” $îìëzãÏr统ÙóâãÓñ{É,XYà ûûûǃ 8•ÉdN·´´„eõRÊŒ¹ƒ´Ukær Žm(Ž4fó×'‹Åf³ª;ºèÙŠ=(áÃýÜb\$á¼µ‰¤IÐÏJÚ8ÈõÑÅÂzBRî”#³ˆ#CÂëÊ,@ jpà &ÊñQMq…–jâÈ…p…”‹ {:»»»ŸO$ÇwîÜ» Ã0 ÃØôõÏôŸî›–Ã{óããûê×L W"B‚j7S–B ‰s ¢½‰~ÝzšOïBÕS%G50ë²Ç’.SÏAÝÂÚN·lB9qÕ‚ÕØÇrÃT|g6 æ&›¼ŽpAÕ™â2U¬e•D‚l Š35 ó‰r‰RXnƒnÕ«ôÛè3*L× ê0IÕ¦!Y ý¯£œC =ÈN¶Àô`¹UoX d+HtHZíÚŸ3T8=è¡5Z“Óâ7öXªˆ(TÕEo?ƤŽ%Û=Lƒ”¯ôvøóÑî×^zóÍí3 dÜÊw‘ÐÑÑ‘•RžºöÚkϯ[·îè5×\sO· !Rîº ø¥nÝ»ÔÑå'ˆ¥Pî€üu 톹Pè‹—TôMД´ÿ„›§êõôCšG ªmžHXÏGbĈ@Å¢4cŠŠöÏá÷³µË­Ö¶w¶é.ÔbÚmùënÏ…ÎÙÐ1þj >»nݺS[·n-Î:Ã0 Ã0 ;61 în¾ë®£k×¼ÿ…–dãÿ ¿vP¤•3 nÁFª=Ô°n>­.0”ªD¡™*t ZY%:ŸDçì©IЖUªäA4jRÎdBKŠKËt\.T˜QÃ1´o¬Ý2°úÿH­I}ü¤iel(ÐYz»2+ —k€âT`: ‰†Iðj¦DÞÊ„±JBÊ®ËX¡ÊSTN)šUêpWËŒ-P)ÈŽ5ÃÌÐ2r¾Xj¤#˜éC¡Ú»ç…9Ϊå/H»¼Š VêÑ¢Ò1„H‹d¨;F:¡°j=´Å±)ƒ²²q²Râ/ÎþŠ=´·££ƒ]É‹¾#^D„“===ÓŸøÄ'ŽíÝ»wk±XüqíRÊ ÚÍk5.vz¦”2üHT™5ºââJ. ê­.”@mí‘&è è2‰2Z¸¾:»å1å„¢š” p‰Ï…Ù‚[/f=F„¡cäØÐÐÐ?>ðÀÝ,Œ0 Ã0ÌÛ'+Œ80zkÝðó‡F÷þ~²¿ƒm„"¤•× ;Óè{ÍÈE Ò’À¾MŽ@-ÐÒêª5#%#9„7Øîæ'Ûõª·É]3ÒÄ‹Jg‰cbGêK•ûT¼°Ãk•V Æ{Zı\ Ò|ÕC^¥tÆ„Ñ’ŠyŠ“µ0sn9Ìô¯†âL3ÈÀ3M´NbZ[$ô°¡™®bÜHf iÈ ®„L gü’ä3²ã=¢ÏŸŠ'ú3DrLËÿè±ÑŸ¹ËèƒÂèŒ!Yzúa¡^'[¤™¬3W )ªûP ¦œræ_É1òüèÀÙÿ¸®vt÷¾}ûXYDT1k1‹)%~üã¯mjjº=ŸÏ?k➺ӟÝÔ Ý„²ë¤:É ¨Y=1ù@}.ÿ®²ÑQ95ÌE^I%ùÓL…öŸl÷BÞZW55ÑŒ¦e/Ћ/د]^Y/ 7=1ô¿ø_éà1æâƒÅ‘%Âç>÷¹æL&s7"~[è^Ï&–0oðlªX¬Y5™¹¿~&ûî@(Ga@ÿÈ–^Áèusá,åyP•;f+°$ŽÔACM­ž†–å5¦±IFO- Ãw%ý]ÓRΪ~_HçIµ÷æ*vÄ•×TÛD KiöŽŽ>µmÛ¶ñ·ûù1 Ã0 3$>ð©ÿóZô’¿ (¿€€×Ó™ì›~°žØ— ÖÔeäÖ™”]˜N2PÔÍ´[ºb‰*Ýt‡Ó—–AÝ&j¹¹w3Òè8‹?±yÜ íšq“R©sÄ=ÔJf1¥IZhR_Œ!ÙÔ¿haFêíÁDh/‘+ÕÓ؉Ь¯aySI¡ÇA×b¾f†W@v¸d‘~N¨Ÿ$+¡b‰=®U.šÀ”NÙ5HÑK¶pcvÝ:fÝñH}V`ÖêvF q>SiJ­ÊŸ=Ì”ÿS3ÿ÷SßÿçÝOŒYpt);" IDAT ëáСC3GŽéþñ|LáGI˜²TQ Ê­ozÜDÈšË]—,Z•<Ö¡ «ê´Œ*“Q¹Þ³,«·:¹ÐÔ%SL$Ìt1h´É鲬鋓ˆý£ž8[01$± iÍ ÎÀBµß¥âGEÀù9î ISjh×pÎO*”!BadG¡PxòÉ'ŸœºrŸ:Ã0 Ã,U¾§ïûßüÕ'ú¦_@À$^ˆµÖA{ìa¾F5vp ˜a‹Ôcó`LéžEØO’¬Ræ*a÷V—@3¶ŠÇ´×¥óSè |4….MA«¸‡l¢ÕôÉ|ÊÙÖXÈ8^œýÕ›&"÷ )7 gÒäêJy.èJ®%^`Ì>…ÁµéÚ4$’‰Š÷ÂV½Óƒ« :FÂàÕ dʨÒbÖ¢ˆ>.tüJœ1Úς긧Ž&SÚh½üÊÏ[[Žô©AC}íX¤+Õ97#¾?‘ƒ?|î‡ÿa(v‡™Eß/Anºé¦Ô­·Þz›ïû‚`]Ì;קøL<èä~-«ÉæÞ…a“5"l‚<[™Î…×í¾ÓR:Gj´"Ous“UBìƒÄ•B.±…ßêJ%vÍÁ»¹Ä•Æ¸ß»ÓÇOs-ášciM®P(l›˜˜xuÛ¶mÙ íÃ0 Ã0—ž+6Ößü‡¿€wƒ”ISúaJf€–‡TµÈÚNÛ1:¸Õ¸c"¶;Á,²²Œ†–ǘŠ õX‹x¨sC› ÌM:Ý·¼G9\W‰qÅçus|tÞœvÂH[lqë‰\;ŠSÏã×gÁo‡dÝdTô-+\8á÷õ-P[_g•ñsµ0=´r£uEˆÊ‰ˆã†”Áiê !ÃRU&dŽŠjR aí·yݨ¢Y˜ó˜1¦*³ê’ÑŽç8+—Êé+ïÝdQ_ž´më–Iþ³±¸açÈdxx¸xìر¾t:}bÙ²eÓˆ¸ î6B¹âTdæâðÊΑ°•oK@«‘º7”z®¶qWµÙŸy*Q—LBÊO˜‹8šåZK Î -œ”ý¥y€© »GŠäÒB[÷ÒïékTàpÅ!AÚåÑóŠºK(UÓìcí«ú½p¥Al ‚`û“O>›™Â0 Ã0Ìå'“È>úÚ[‰ÿþ×mµ¢nL±AÖK@œ{vë×ö ÔM@þ«´œ¥Ó™þUO ôÁ/ñ‘z@éú³*9ÒÆì‹šúNhI¹1ÙWU´cÙ^bŽ›ÑWtü«.9“ƒ}:1>õø ?þF–,Ž,qÚÛÛeww÷hsssçØØØ™ÚÚÚ|$Ž¤gIX©Ž:6ÀoÌfקhV ÞK9WHúË£]ÙèTOZXòhšºTüP¡äÉ…œÌa#…ýý¾·¯Ç'fˆ0âŠîë¥vuÑ÷ÊâŠ%¤sŒõ3-Áq—3—ãJJkÂõö÷÷¿ÔÙÙ¹ÿþý,Œ0 Ã0Ì<¢»»;8ß}¨Ç¯_óÊÌÔ¹7j[²EÂêc$nû¡¿!´ `Ü#ÎòÄßÊ°Ê>ì< 7Ã`¡Ì¬eém$ËQBû Î¿°]+ƽb¯ß”óy½,žPÍÈÚPI·•:/œm×ù#j¬H;ì”$…©4…šòû"(ý«© µ¬zÈŽ´ÂÌ` §öñBûX õ9HëX)AŒT>‘m«tz ©ï1ŸÙ×j12êl¤§ÏV¸ã©«§4?†ÂÈ›CçN~£}ûã?9úÆó\J³„`q„)ÑÓÓS8{öìPKKKW__ß™ÆÆÆ)ß÷RÊÚ¹8IG Èt"(6 ”iÐz¼¹&«?ínu¦ý !žp°²&<Ï<Á ê¸õä\%âÌ„€“=~bßy_ôäËU£Öölj#ôu%h¸Î¼ê–ÜP1„>Ṙàhºp{{_êíí=xìØ1.¥a†a˜yÊH_{vèìñ./Ú1Ñj_òµ#ˆÞ‰‡Í|×=DP Â‡~ðCÆ4@’MA–d¾¢#ÌBüthÊV*ß2#¸ :aÍ}?fÔ²Æ6ÓRê˜P.d’N«• º–Ë¢¢.AæPœNCOAX•L7Ca|%d‡!(xÁ¬Æˆ]´g0ƒ_ãhvCaÕ¤='8‰2tÉ4ÀµšÓLj8ö[–ë (ßîëøú‰Ý[ŸèïïgÇȃoz‹P$éïïÉd2gÆÆÆN­X±b&‚!Dý¬wêLbMøcYßô²¾ j0RZ"¹ªÄÕ˜j;(±Jºïקҥ²´.Ñûê¿êâ+¥,¢71ìãÁS¾whÌ÷†‹1*Hµr÷u÷g*žèÜ”*ÓP· u•ÐÒ™*¢I»ûûûŸïíím?|ø0 # Ã0 ³=×=32pú´Èfö 6û{›±i\ Ù„­áp ´Ž a–^·z:=†ŠsyŸMhL†Y¬8£ÞÇuRŠqKŒ58ÆôR1‹þ)ÒûIçsZJTÚªä¥H:ÆŒOƒÐI’€ [ "XA¶@ å¨0Û§ :C—(ÇDçÈGiŽ©3vÓ¢˜®Mû¬­®=¶ E®€£·E¿@6$Ú¯Òÿ$äAÊÝ#çºÿ¼óõŸm ã.ü‰3‹ G˜XÆÇÇ ƒƒƒ£333§&'';W¬XÚB…?ÅGlvè…1@,ä}2“ðû³ž8‡Â›òY/‚ )Qý §Oyƒ:B´@b¦¯M&Áœ#ºfÖ s…²[$("Kˆ==¾wb 1ç‰c.ӹ᬴† %ôH¿W_]K¦“[–þœÈd2Ïd2™Ž½{÷æßÙ'Å0 Ã0Ì•ftôÜÌĉ·ÎNþñvgë^kJ4„a× Ä:sK,èÜ–€5†RI¨43„Œ¯Ê™ …åÐP7Þf•¦àY?|rø¸ÒJUè»v)`¼Žàì‚ HiŠZ–ž†t|Ñ6ë0‘±$AÊ ÑFÚ]¡EŒ@@"Ñžçë#dcÑúLbê}@•vÇY5ô‚Œc¦b)&+Få²è2"¤m”ÌJ7Xùa˜£N“•/œÛröГ¯œ:ujæJÿ^0óG˜Y- ŒåóùŽúúú³¾ï‡u¢ÍRJÁ¹#³c‰B²e‘ä|>áŸE!Æý (•Û¸õç]F5—´ä†Fª–œ#É(•X-±¡üMCìëõ¼Ý}¾8“-‹ —·´†Š!r¨÷-Q‰´íÞ ·ùðÐÐÐ÷ÜsOïc=V¼ÔûÀ0 Ã0ÌâK_‚éááìxÇáÞšVñjmíµPÊU€x­º?±r9ÈÿLÙpø†tJ¡Ž2MÙ„AÈH"v¸®ññ…8P¬ÐT³ TëâØ a¤–¶C*QPÙBâ¶(¡\’¬³òa1ŸÐŽ<m~ÝŸ£…€dM=x‰„ 7%Ó¡%4ÐWL»^­SAÈônVóVº}ÜOAO-Pë:–ûG/FçêÅ8JÈ'n è•0·îç™±Ñ?X.¿¹}ûv~ø¶„á»[æ¢x衇Óéô]RÊ !VñÑ‹õÅ06»±(±¶XlmÎLßœ*äß%‚ N]/¤=1úb(­÷ÂÌ‘åõõP›J[Ým0š¢ÅM!œ:•ôwzÞP1æf™×%R 7dU Tø ­ît )eA±wxxøçÛ¶m§ÛÏ0 Ã0Ì|CâæþóÕ5-«À/ €›UùHEˆ'{Kœpœ D$Q¯iA$Z¸Ý9ƾ-Šs»ê- µ¢.zÕX¨;Õèæ<ä¡Pe1*7ƒ•9R¹ä;šÏ¡„+‡Ž¸+èBÉÂ%9ÞaÜ`ݲVH¥kÜÔU«Ån\Y¸ZÍó0Ò‰´„¦Êü9i `é7ÄZS¹:K\Š2ÙdIBQ·žp½œþ ¹ÿçÉü‡¾°“cÕ‘Y°8¼->þñ·ÔÔÔ|ïÂr 3*‚H¼Æ\nõ²ÌÌm‰|ázPO8¤@ëâã^¤ XÙP©DÒúEÖqÄ─Sg|oÏÈ%FÞqBJ FHÓ.ºàûþK333/lݺušÏ1†a†YÜ|à7þ÷ë=YóoAâÿëM5‰Ê(ÿ,µH!Mé„¥$”E ZBˆ-xØ™"–xŽGg¬½Œ¨Uo±Æqb *Õ!÷÷‘“BïôIínqsIâíËâ‹pD ¥ ”ïù#Ð4U&¢íPß¼’á80¥+ÖþëåKíÚz Öúë¸K«å1Ä”»C’6»t[lK™Wù&j ]:LyQÙµ 3 àë&¾þô£_á?+ °8¼S>õ©O]—H$~7ÊrGÆÁzºP% Ô/ÊTs.»¾n&{{²PX!…$O5$½ EðCAae]=Ô$“¶‚_vŽ¦ž>ë'ö úØ9„‘8É\]%#š¸ËBäŠÅâsããã¿Ü¶m·êe†a˜%Äýÿû÷ ‘ø¢ùIhÔ{ŽHí´U q]TŠ` h®sBBå{j« Å`‹-N©M8ÖÑ·æÆÁç‘Äí@·I¯ì¤IuZx ¡¤ÒgœØ+!_QxPÛ²RÉ4qpØ.kO-§ˆÛ{˜Ni$½z"d #þ˜8¦Ãa4¤Ù0du¥×·I RSòO±ÿöS?üÉ¥õ›ÅÌgŽ0ïˆãÇ­Zµê­t:}jÃ<ÔÉI T±cº‹™„?4ðÏ BV²Ö“2¥ŠW…³ UÓ–Ôx^ù×X—Ô§=qæ´ïí00×àÕ‹%n¹ô5ÚÚ—B»ßÄuƉ¾f …Â3ÛYa†a˜¥Ç™c»z“74þ¼N,oGÄf pm踵o‚I€‡Äé8 H^[Y!nMœ«Äv›T #Æ`/†”«Ä=®XwÅ.G ™Ö8LP/CJ’»BœÕ†ã:žE·&®,s Ý©šZÂÓë­p?‘‚®Z/o6JŒˆ¤ [j1®hâ2V”úá#–ëÄÍ,‘iP~iÂô¹ïþ§ÉÙ7”Yj°s„¹$H)ñÞ{ïmX¹rå½¾ïß+„h C[—òÑukI/F¤¨+›g¦oNæòëý@6€>Ñ•›¡`+!4e5Ed§…8}:áíB¼,ÁjŽK1}ÔÝf8›Í¾|àÀ,Œ0 Ã0Ì&cnúÈçW-«_öyáù¿ƒÂ[/!ðMWÐVúˆå¸ YîâbÞq…(…“#g¹O@VÈ%f ˆ;‚txÑs:ù®“ÄT§¨ò Œq¶˜ðzkÕt³`JaÀt@xаl9$Ri²7’&I^¥‚L‘ˆ:bte”4Y$Jà‰=ŽŽ?ÄéTc¯Ð»l%MJœÎÍL~srt⇯=õUvŒ0°8Â\R6oÞœhmm]—J¥B‘äDl,ÝØ/f b½nùuù|sm¶pC*ŸÝàIÙ„Å ÔJY)ø«!åû¥!Aää˜çèõ½7Ç„N…“0KJ9”ÍfÛÞxãÝÝÝÝÜFa†a˜ë×?˜^¶á†»kë—ý¶çù¿Š«%@*r‰RIq9-h9Xß±Œ#Ä)‚±³1hùS¦â¶ö-|¦‹¤_‘!s'F—KÅâ‰)í‘®Hàd˜¸cLZª­E"$„e5õÍ+!–n[bŒ½ºû%29Î>zßd…$V,¡ë¶Ž³t?;IJŒ$]’:z”Ø•™þ/û·ýøGÃÃðÏÄÂe5Ì%¥¯¯/èìì]³fMÇèèèÙúúúð¯R ÛúEï$¹`¸ ¼Fm“áe9ïûÓS©D_Ñ÷ 3@RHH€”^øg¿> Ëj‚Š„wð´ï½5…pEƒKU™ŒûŒD¿ÎÅ‘ìo€ˆç3™ÌKx…†a†a(££Ý…s]Î$êó¯&rÓ Ë‹°ê±ü NµË¥ßÇ49Ñå5j&Rž\¢"_$¶8Æ‚–×IÃü×´ŽÄ¬¾T]öBÄ*ŒX¦ §G—¢X‚‡i1ìs¸ÂHYËjêJ¥/q%=ª„Å7ÈrÕñtŽ ’ãLKÐq†(qÉ-á¡b Cë’'º}ZÁbòÈÔhÿ7öïùÛôv²0ÂT…Åæ²ÐÝÝ]èééô<¯kxxø|sss=D¬[ÌŽ¥ÙÚ½Í&œÌ6}¸¼¼çMM'çó¾ß12ðéºTºõDw¯/öõ{âTàŠ÷fW¸O#¢}På4nÛ:¼Zjl,eïÄÄÄK‡:ÀÂÃ0 Ã0ÕéëËôëH¼# x¤ÑošF” l-™HJó!¹aWnLJnÌUaˆã‰i¯ Ö2uf‡#©mçHÜC2çgê*Ñ%8dWd5ÅÚ,ZFDÒˆ:·$r ¥¼>! U[¯sí¬•jeHÖ±‹õ‡‡Ôû‚`"j»ôg¥òb´ˆã,U:=wÜZ‰y Á›“cß8³ëÏz;Yaf‡Ëj˜+nØ°¡WÜyç›`c±X\%„Xtç_5qÄ °ª& TÖ›Ú‹OÊD²Ph®ÏæÖ/¯©Åþtòè¤ãÁUjÕ ³t©q_§e4Ñ÷Å N†ÂHww÷ñÇç®äv3 Ã0 ³€‘¯½îîk ï¹á¦›ênú5Á'%à-&ŠV*–€¡Z̪ÿ¸e!Æq‚ M݈»P#¾Hãzf%ÚÉ`ˆðÁ Qé d}Î{2æUNDZ­l²CZ¸e©B`erJ¸¼†ÖÕàû¾àj©4Ä.h¹‹šÏ*É!ÇÄÁÞÞY=;¶ªb¹G@9|mjlðoµ=õòùóoNU[ ÃÐs—a®kÖ¬©Ý´iÓªúúú;¥”›±i!ýŠZÓ‹ \G4¡Ä (1OP‹‰šdsB,ÈÐR!D(ŒÿE&“9ÛÖÖV˜›Å0 Ã0ÌBCJ\wí¦æk6ü–z¿€¿… ×€¾ç–v Ð{}šbÊ6Ê?Úùîk Þ SKá¨Õ­£²…0TíT™¿bqîƒ73«)Á±æ0‚‰µX±FD[W‚ç'¬yA¸¦\È u¥ÇÓhvƈ*ƒQ•©Î1 Ž•½@pòVµ³&*ëÈI‰Oç2ãÿùÌŸä‡oÌ\á–«Ì¥··7óÜsÏuïß¿[¡Pø8(¥\p°æ"ŒèšÒªÍ£^s#nÛÛ0¹4ð¼ÜBFH‹^õµ{“Éäß?ýôÓ§Ya†aæmƒ({zò®Æ]ío=ñåäç%âß#”ÇV¹Iù&Û¸êò•ò+öM:(aD/-r”ÛÒUÂqa˜•—k~H92)i©´„TŽ-éXÒ–UìÐT3®ÔnåpQsè÷l׌»P¤¹%znúZÔI†,i§ewÔ8W D>ž—_¼÷]Á>F˜‹#ÌUå¾ûî«Y¹rå]žç=€ˆkcûß •ÕÄ9GæºÜD"1‡)çˆX@Äg=ÏÛþøã³Å‘a†a˜KÎú;\¶æº÷>"„ø7ñ)_Z]MHí ‰æÐn’H¶Ðî¤Bíd㦌 qR¸å<à”QW” T8'*6Õjå+I—$NˆÊcl¤.k±vØ9"ìVÓºüD²2¬U»7ìíGåQB’Ó5ˆæ¦Ø®©«Én‘ÑÿÏKÍA>#kÙYò+™™é¿záG4vBäß,æb`q„™àG?úÑ• FÄ÷"bÍBýTf+¯©Þ'nˈs¡Ìwq„fŒ„ø¾?S,C×Ðö­[·²’Ï0 Ã0Ìåã‘G¼»3+Þ]›¨ý·ð9\f³œ zïÈØN ·'*3 R¢ƒ‘›$pƒCc\ÈÚ ²B±:ÕX÷TÈqĵli7•±“Ah.Š §áCCË ðýDù5-¢˜Ö?vfˆ0BY´U/DÝ.ÊQB„ioÙàjY/åi-ÈÈñ@Ê/×ÖÁw¶~ó‹W´s#³x`q„™Oˆ‡~ø]žçÝ/¥¼MáÏçOçíæŽÌ!Oä‚∺0.0çȸâ…‘‘‘Û¶m[9) Ã0 Ã,@yÄ{ wýGñ÷ä¯!`åJpr4@gYÐÒzïÞŒG9#nبjÁ«2Fœ›kù@ÃPi¹JxÓ&Ãà í˜Ðër0e-hÜ/V‰Ý£8OJ·|¨Å‘DÂd‰è-SûS‚ÈÎ8Ù%F•Ñ¹#ú˜šÄ+oDR5G-ÍöÐÌSœ¾:æ¯í;[&ùW”y»°8ÂÌ;zè¡T*• »Ú|ׇ-€çÛ6º.jï_hšÙÊlª (êõ°½ši±6¿AÄ1!ÄËx•k?†a†¹Üþ«ÿW]}:÷)!R_@÷B }cQ†¹I§9¢ºäCßÙÛ%9ŠÒ¸. ­‰«¤»ÒÌrJá¦áziÍSc…±ê}@»Ŕ٠v™X˜Rc9VÊb‹ð|¨o^^ d¥KÓæêøÔY¢wÐtÆ!Y*êØÒ£†Ñq ™Õ½1&¨Õ’^̾÷R~óõ§þìÑóçÏs¹6óŽ`q„™·|úÓŸ^–ËånO&“÷ !V†U‹áÓrÛö^($®¢’•… Žxž7–Íf_Þµk׎žž¶92 Ã0 sU¹ïWþýÚbjú3Édý¿FÏ»d ÷øf|‰ `—ܸY%´?®)µ1¾Úu… -A$ØŽ µlé8WÐH¬.5Ñ–¨Œ‘Š1&‚“Ý¡²:D´…R—½„ß/YrŽxžzk÷‰4ÛE96$XÕçVæ¨ØåIv©ŽÊ.¡'f\¬ƒY{‹ÁÌ7w¼üÝïœ<9Æ¿UÌ;…Åf^#¥Ä}èC«kkkßW__G-óÁIr¡l‘·S^1 88W< ûÏWëˆsµ‰B°Fr¹Ü«'NœØùæ›Ü_ža†a˜ùÁ–-[Ä¿ìzOÂOý‹šº¦ÏâõRʤ¾×%)庰‚J¤$·ƒ[JãX?hƇ$ïëV»®Q+_U…)B­³8ËNqAx‰Ræˆð¼Øn3ö>’cÖÖè=@CX©s'Æu¾Ýѱ{|ö#Â0‹#Ì‚#,µ¹á†š6mÚt}2™¼ÂðÖeóa?æR*'Ž\L÷šùœ9"„(Apbjjêʼn‰‰Ž¶¶¶Â<Ø,†a†a˜ –Ú|ÿûO\³áîò~ ⟀Ÿ€un0«[:cÆtJgˆ÷„h1% aw£h¦ µ_T„±FË´JjÀŸ:Óéiõƒ¹(Õ-‡ÏO–Ä!<¡˜R‘+2KÆBTKdÛ]âKâ‘övšÎ?¥—óäËÅ ÷ÇÛÚ^éèà·̥‡ÅfÁòÈ#x/¿ürí}÷Ýwb“çyw@Ã|ߟ •×Ì&Ž¸BË|*«AÄ¢bO.—ûåÆû¶lÙ̃Íb‘F!~IDAT†a†¹(6oÞœèém¾ñþOÝ#‚Ä'à7 sŸNºÄпô!˜ÛmEJ+®•Š+Ç”ŠPÝ:'ˆh#’õ‘Ö¾¶4Al1Ñ2JΑe­¥®5±e5®ZvÖä@¨Zžj}†Ñ쟳º“¯rDØÉ&‡ Ÿ(N~[ž|é0?|c. £(ÃÄÐÞÞ.3™LîsŸûÜÐÓO?ݹnݺSA„çt3"^vÕàbƒPKiäÄ-BçûbòFÔ÷á×Ð9þ› bØž÷åL&óâÏ~ö³Á|ä#sõŽ2 Ã0 ÃÌ+úúú‚©©‘©W¥»íÞõÊŠënÙƒ D¼ÒHŠBÐ郮0¢r‘v´ª«ŒëÆ0á§ÂŒ­ bŠd%1­hZîše’õ*0¢Úì†yúèyJ×ê™­øµ<f‚Ί1ú–ŒoIŠY†Hh@‰íp‘2€ø©ìÔ×>¼¾ØùØcù·…¹\°s„Y4„å6k×®­¹ï¾ûn ‚àAD\{¹Ïñ¹„¬ÎZƒZeš¸R·',© Å‘yЭf\ñ\oo¶6Îa†afQ–Û|ïÕƒÍnø€÷û€p‡muÊA”íÁjÝ[m¼ã±JPœ÷MûÜÙG¸VIJìÏŽ«„v‰Á²s¤¾©µ\¾m=°S+ 9,åýºE¯´;ñ8%5tŸkÄvÜ@$ŽHçàOŠ)ÿ¯ŸùÖïODæ²Áâ³Á»îºkùúõë?àyÞûƒ ¸"¥6síF3×åÄ "nˆëÕn勈ÃÅbñÅÎÎÎûöíË_µ a†a†¹ÜH‰>ù/ßµ&¹ê_Âo#À [3¥,û~Ò´`A_•Ñ¼‘ð@Ë[¤Y8nf{ìèv·±Û»òˆ@D" õM-¥V¾@7 í6ÆdcŒ¨CÇÃZÌ![‘tBvØëêý8B~-5?غõ‹Ó|n3WG˜E˃>è¯X±âÝA| 6H)/K Ê…BXç2ï…œ#q˾ڬˆ8†ˆ/yò䞃²0Â0 Ã0 C8ßýæPkâ†íÓÙÑ¢¦~ÊC¯6†─¡ˆ-Zß[Ã;ZBöxQ¹;¤š‡"ÕƤ2Ê@Ѿ• ^" Ét-]ôVi'Ýä²Ð!ˆ¢; SAÇv¾m+ ùÂ7ûöïøÑá7žæsŠ¹Zp泤¹ï¾ûj„Z[[oBÜ*„h~'Ç£Z°ª›"îÎ3[9Mµ<“0oäJdŽ!8™ÏçÙÞÞÞÎá« Ã0 Ã0Õ‘Râ7¾·±ví»XÖrݯŸ@ÀuêÞË1E¢™ÄeŽ8ÐðUÚê·J{ß8‡1m8C5õ}"]õMÍf\«-q(ç šljbï„)Ó‘( (qG¶0ù­Áƒ»·íÛ÷\®Í\U®l`ÃÌ3vîÜ9ý±}ìH{{û3SSSÁn)åä;ÙʸÕj«q¡¬³…±^鶽Bˆ¢”r>Ÿ ±0Â0 Ã0 3;¡ãöäÉ}cïY]|öØÁ§ÿx&3úÅàG€8PšQûT‰4¢Ó˜–¨¨×M¦y¯ÊÑ j%Ë° :ûôò̘TχŒSk”[Dª\Õx'5FÙ#ºLó2€ŸÌdfþhðàO±0Â̸¬†Yò´µµÉáááìæÍ›ÚÚÚº®¿þúÞÈþØÉ‹9>oGĈ+›‰kõDP©´SÆ?x' bVJùê¹sç^xßûÞwî;ßùN°Ôφa†a˜¹ÒÞÞ.3£“ÍélÇé£v4_Ë!”—¬Àšòb¤•êᆚšSÚîÖ¨$¸á¬–D½dO阬²™ð{ÏOB"•*gÛi|«*?‰¥Ñ½ å7óDm L²øèL1ûÕe™ý‡_xá…ŸTÌ|€ËjÆá‘Gñ^~ùåÚxàÝžçm€[¥”–8›€q1T+·©Öî—B[ù^†PÖÏó~ÙÝݽgûöí“á>O†a†aÞ>›7oN]¶¶å®†›EÊà7Å' IãNKšG:(£ðýٰDz$xÒ2XZú‰IR­V.“HÕB]ã²hÝÊñAÌ'X±ÐŠÚ%˜¨VÆ:Tå™bߖřǞùá ò“™O°8Â0Uزe‹xì±Ç’÷ÜsÏíÅbñAlBàlBH\~Èl#qóÇÍG(Œ¸¬—H$ ‚à…ÎÎÎûöíË¿Ó…1 Ã0 Ã0†ðAÜ‹““u›’›>]ù»ˆð€Ðµ º‹‹¥ñcØlœcª_“ÒrŠ˜ñ!ê2#‹È(ÿÃä–„™#u Â^Ä¡‚¦ñqŽHˉ¢ÄóºQT¤”§!_{ëÅo}?ì"ɧ3ß`q„a.@ªõÙÏ~¶9‚û¥”÷@£šc6Äe.A¬³‰"îŽ:G.!£A<ÛÙÙ¹‡…†a†a˜ËGø îÉ=×Öûøoñó€°ZZ]_Pç~”µAMKCV¸6Âr˜ ZB«^nE÷-ŽÔB]Cä1¾àjrH´ô@¶\V¸F){¤€¯|úë÷ôôLóiÅÌGXa˜9²qãÆä-·Ü²AJùaD¼YJ™¸ÐœSnãŠ%Õ¾‡H ¡ÝjÞ©c$jÕ;Á‹»víÚÅj>Ã0 Ã0Ì•aÍæ͵ë×|èÏ÷þüPÊ#©t‡D57hÚêÚ·‰•;Ž%í) Qù6€T TJΑú&@Ï+ÏDA‰–ke˜8a¯@²QÊ™³ò,bðõ×^zô‡Ãã|Z1óde˜9200P¥˜ù‹# ó6éîîžìììoJÂtÀ·F·=úÆ®gÙ1Â,Xa˜w†ìêê»óÎ;Oöõõª««+ !B¤†þ~Í&r¸A«ÔI7%GÞQÛ´žb±øÒÑ£GwïÝ»7ËçÃ0 Ã0Ì<¡½]ž?ýÖÙkomd¢koºnEN¨AĦ²“Ä.« m­e‘sD Îx“.B¹Nüdª,ŽÄ Ya:Q±ºÔ§H8P„ü7Îœ:øׯ·mäSŠY(p +Ã\B|ðÁôøøøºë®»îÏón ‚`bØšmní{]ÇHµ’™wÈŠˆ8:22Ò633ÓÙÖÖVàs€a†afþrÓûj„\á®U×ßñi!áW%»0×Ò÷„%2¦D§{¦ë¡¦^7gŒÍP¡°´Ÿ ÌÈçsræ[ÇN<·½»­þ™‹# s‰ [ÿ®]»¶æ†n¸võêÕwK)7 !Âש¸NæÈlnÊ;iå+¥Ìxž÷úÀÀÀk÷ßÿ¹-[¶üù3 Ã0 Ã̱äµ×njnX}ãæ–µ7Q~W‡CGåè¨tŽÈ¨[Mä4Á*gHpkI©kˆ b-·ç5ᯒÎ; (ÿ6Wœù«¦ñG¶nÝZäSŠYhøü‰1Ì¥%*YÉüÞïýÞ‰¿üË¿ì½ûî»744„"É­q¿sq™!qm|«M¯ˆs™Ðé…£RÊ×víÚµ³³³s<ÚN†a†af݆¹ïÖ—¶}íà-­›_J‹ägÄ'! üL·â A7 MQŽVôÃ9*œHIJi¢¥II:ç”^8#%~/w>ÿƒûïΟ۲e+?|c$ìa˜ËÌ#<â½òÊ+é{ï½w“ïû÷À†P«€YÊa⑱£¢¬¦ÚòBažß±cÇžîîn¶82 Ã0 Ã,p6nܘ<½bE㦦»ÿ©‡âwà, ¥ãí Bˆ%xXc̲îQvŽÔ—Zü–Ç–¤°ZªQL΄üêîŸ~篇†NLðÃ7f!ÃâÃ\!¶lÙ"vïÞ]_WW·)Œ' ‚àˆ±3RfËQ™#³ #a;D*‹¿hooßwøðáÞ Ã0 Ã0‹‡ðA\÷ÔúÉü€ü?q“^¥å6 í*ªzèv½éÚú(sDµò¥Þ X^v²þtß3ñ·½½½>¥˜…‹# søÌg>ÓŠˆ÷ !> ¥lœ­Œª(45Ž¨Uï¨âvŒ0 Ã0 Ã,nÂgbïøÖ§>ˆÿ¬¶F•‘câÊlÈøÐ9 $®0Bƒ Ï¿¶÷—þhðر >µ˜ÅÀÛëÊ0Ì;â‰'žºýöÛ·e³ÙïÀˆ8=[GŠjã«Pó8®á#Åb±­««k/ # Ã0 Ã0‹›pü·çÿ¢Ë›xýË…bñŸÉ¶"À¨7FcH»…¯SV”fÓ1f”3‚(ÅÓ‚ÿrdÏ?ü #ÌbââZ]0 sÉhkk“ÇM$Ç'…-ˆX«DK$°¸’÷}…"tŒŒ‹ÅWOž<¹sÏž=Óü©1 Ã0 Ã, º»»ƒž;ϱðÒ²Ö ýRסÀæÒ½ŸUFc"[‰D~2 žŸXž\¿Uj¼Ø+Q|§û­ç~xâõGø”b,Ž0ÌUæüùóùcÇŽN§Óy!Ä2H ¢ ¢ˆʪÚùX!­Jy¹§§g # Ã0 Ã0K“‰Á³Ó§ëcû½¯Öú5¡Ky­¨A,…¶Fg$؉«¡8’‚D27uç ôÈÿï\×Îï|å©Q>­˜Å‹# 3O8wîÜÄm·Ýv¢¿¿ÿLMM !’þó¨k„Š%JrI ¥‚àØõÜsÏeù³e†a†Y´µÉ‘“úWÕßÖ–Á}©DƒBÖˆšÒÃ8-Š(‹H"•†D"¡ö‹, Á×Îõÿá¾gŸäSŠYŒp +ÃÌC6oÞ\‹ˆ×®Y³æöD"q#¬…ê$¡­|±º(ÇÇÇ_˜šš:ÚÖÖVàÏ•a†a†¡l¼ïã-AÞÛܲö=¿Žˆ Ä›$BÍ]­­o‚dMm©•/‚Ìvÿªì9óü¶mßæ‡oÌ¢…# 3éëëË÷ööµ´´t={¶gÅŠÓRÊ4”­%»éT“B¼uäÈ‘g?úÑv<öØcEþL†a†a—žÎéÁÞ'½šÔŽ‰ó=ûZ׎€ÄF@¹ ýð!\èñüDXt3,¥|º8œÿÊôÙm¯¼ðÂOó|@™Å ;Gf°nݺš¦¦¦µ7ß|ó¡è­žç¡çyã°cÿþý»?>¤ZÖ3 Ã0 Ã0Ì…Ø°asSîæwßy}rÝ?C„OâúºúeÂO¥Ïy|ÿÿÇc§O¼~2ê„È0‹Gfá€ëÖ­K/_¾|Ý7Þø!Ä:ß÷·¿ôÒK¯Ÿ?~Š?G†a†a梑›o|ocË{6Þ}MbÕïÖÖ5Ý‘J¥ÿÛ¾g¿û·½½Çù€2 Ã0ó)%Þ~ûíu?üðª7&ùSb†a†aÞ)[¶l·¼ï3­Ÿøßþüæ5k6×òe†a†a†afI>ˆãOža†a†a†a†a†a†a†a†a†a†a†a†a†a†a†a†a†YÔÀÿÝEÑ@Ùû…IEND®B`‚SpFFT-0.9.10/docs/requirements.txt000066400000000000000000000000101357247005600167370ustar00rootroot00000000000000breathe SpFFT-0.9.10/docs/source/000077500000000000000000000000001357247005600147645ustar00rootroot00000000000000SpFFT-0.9.10/docs/source/conf.py000066400000000000000000000125601357247005600162670ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. # # This file does only contain a selection of the most common options. For a # full list see the documentation: # http://www.sphinx-doc.org/en/master/config # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # run doxygen first for code doc xml generation import subprocess subprocess.call('cd .. ; doxygen', shell=True) html_theme = "sphinx_rtd_theme" # html_theme = "bootstrap" html_theme_path = [] # -- Project information ----------------------------------------------------- project = u'SpFFT' copyright = u'2019, ETH Zurich' author = u'ETH Zurich, Simon Frasch' breathe_projects = { 'SpFFT': '../xml' } highlight_language = 'c++' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags release = u'0.1.0' # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.mathjax', 'breathe' ] # Add any paths that contain templates here, relative to this directory. templates_path = [] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] html_extra_path = ['../build/html'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. # # html_sidebars = {} # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'SpFFTdoc' # -- Options for LaTeX output ------------------------------------------------ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'SpFFT.tex', u'SpFFT Documentation', u'ETH Zurich, Simon Frasch', 'manual'), ] # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'spfft', u'SpFFT Documentation', [author], 1) ] # -- Options for Texinfo output ---------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'SpFFT', u'SpFFT Documentation', author, 'SpFFT', 'One line description of project.', 'Miscellaneous'), ] # -- Options for Epub output ------------------------------------------------- # Bibliographic Dublin Core info. epub_title = project # The unique identifier of the text. This can be a ISBN number # or the project homepage. # # epub_identifier = '' # A unique identification for the text. # # epub_uid = '' # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] # -- Extension configuration ------------------------------------------------- SpFFT-0.9.10/docs/source/details.rst000066400000000000000000000135021357247005600171440ustar00rootroot00000000000000Details ======= Transform Definition -------------------- | Given a discrete function :math:`f`, SpFFT computes the Discrete Fourier Transform: | :math:`z_{k_x, k_y, k_z} = \sum_{n_x = 0}^{N_x - 1} \omega_{N_x}^{k_x,n_x} \sum_{n_y = 0}^{N_y - 1} \omega_{N_y}^{k_y,n_y} \sum_{n_z = 0}^{N_z - 1} \omega_{N_z}^{k_z,n_z} f_{n_x, n_y, n_z}` | where :math:`\omega` is defined as: - :math:`\omega_{N}^{k,n} = e^{-2\pi i \frac{k n}{N}}`: *Forward* transform from space domain to frequency domain - :math:`\omega_{N}^{k,n} = e^{2\pi i \frac{k n}{N}}`: *Backward* transform from frequency domain to space domain Complex Number Format --------------------- SpFFT always assumes an interleaved format in double or single precision. The alignment of memory provided for space domain data is guaranteed to fulfill to the requirements for std::complex (for C++11), C complex types and GPU complex types of CUDA or ROCm. Indexing -------- | The three dimensions are referred to as :math:`x, y` and :math:`z`. An element in space domain is addressed in memory as: | :math:`(z \cdot N_y + y) \cdot N_x + x` | For now, the only supported format for providing the indices of sparse frequency domain data are index triplets in an interleaved array. | Example: :math:`x_1, y_1, z_1, x_2, y_2, z_2, ...` Indices for a dimension of size *n* must be either in the interval :math:`[0, n - 1]` or :math:`\left [ \left \lfloor \frac{n}{2} \right \rfloor - n + 1, \left \lfloor \frac{n}{2} \right \rfloor \right ]`. For Real-To-Complex transforms additional restrictions apply (see next section). Real-To-Complex Transforms -------------------------- | The Discrete Fourier Transform :math:`f(x, y, z)` of a real valued function is hermitian: | :math:`f(x, y, z) = f^*(-x, -y, -z)` | Due to this property, only about half the frequency domain data is required without loss of information. Therefore, similar to other FFT libraries, all indices in :math:`x` *must* be in the interval :math:`\left [ 0, \left \lfloor \frac{n}{2} \right \rfloor \right ]`. To fully utlize the symmetry property, the following steps can be followed: - Only non-redundent z-coloumns on the y-z plane at :math:`x = 0` have to be provided. A z-coloumn must be complete and can be provided at either :math:`y` or :math:`-y`. - All redundant values in the z-coloumn at :math:`x = 0`, :math:`y = 0` can be omitted. Normalization ------------- Normalization is only available for the forward transform with a scaling factor of :math:`\frac{1}{N_x N_y N_z}`. Applying a forward and backwards transform with scaling enabled will therefore yield identical output (within numerical accuracy). Optimal sizing -------------- The underlying computation is done by FFT libraries such as FFTW and cuFFT, which provide optimized implementations for sizes, which are of the form :math:`2^a 3^b 5^c 7^d` where :math:`a, b, c, d` are natural numbers. Typically, smaller prime factors perform better. The size of each dimension is ideally set accordingly. Data Distribution ----------------- | SpFFT uses slab decomposition in space domain, where slabs are ideally uniform in size between MPI ranks. | In frequency domain, SpFFT uses a pencil decomposition, where elements within a z-coloumn (same x-y index) *must* be on the same MPI rank. The order and distribution of frequency space elements can have significant impact on performance. Locally, elements are best grouped by z-columns and ordered by their z-index within each column. The ideal distribution of z-columns between MPI ranks differs for execution on host and GPU. | For execution on host: | Indices of z-columns are ideally continuous in y on each MPI rank. | For execution on GPU: | Indices of z-columns are ideally continuous in x on each MPI rank. MPI Exchange ------------ The MPI exchange is based on a collective MPI call. The following options are available: SPFFT_EXCH_BUFFERED Exchange with MPI_Alltoall. Requires repacking of data into buffer. Possibly best optimized for large number of ranks by MPI implementations, but does not adjust well to non-uniform data distributions. SPFFT_EXCH_COMPACT_BUFFERED Exchange with MPI_Alltoallv. Requires repacking of data into buffer. Performance is usually close to MPI_alltoall and it adapts well to non-uniform data distributions. SPFFT_EXCH_UNBUFFERED Exchange with MPI_Alltoallw. Does not require repacking of data into buffer (outside of the MPI library). Performance varies widely between systems and MPI implementations. It is generally difficult to optimize for large number of ranks, but may perform best in certain conditions. | For both *SPFFT_EXCH_BUFFERED* and *SPFFT_EXCH_COMPACT_BUFFERED*, an exchange in single precision can be selected. With transforms in double precision, the number of bytes sent and received is halved. For execution on GPUs without GPUDirect, the data transfer between GPU and host also benefits. This option can provide a significant speedup, but incurs a slight accuracy loss. The double precision values are converted to and from single precision between the transform in z and the transform in x / y, while all actual calculations are still done in the selected precision. GPU --- | Saving transfer time between host and GPU is key to good performance for execution with GPUs. Ideally, both input and output is located on GPU memory. If host memory pointers are provided as input or output, it is helpful to use pinned memory through the CUDA or ROCm API. | If available, GPU aware MPI can be utilized, to safe on the otherwise required transfers between host and GPU in preparation of the MPI exchange. This can greatly impact performance and is enabled by compiling the library with the CMake option SPFFT_GPU_DIRECT set to ON. .. note:: Additional environment variables may have to be set for some MPI implementations, to allow GPUDirect usage. .. note:: The execution of a transform is synchronized with the default stream. SpFFT-0.9.10/docs/source/errors_c.rst000066400000000000000000000001021357247005600173250ustar00rootroot00000000000000Errors ====== .. doxygenfile:: spfft/errors.h :project: SpFFT SpFFT-0.9.10/docs/source/examples.rst000066400000000000000000000230251357247005600173360ustar00rootroot00000000000000Examples ======== C++ ---- .. code-block:: c++ #include #include #include #include "spfft/spfft.hpp" int main(int argc, char** argv) { const int dimX = 2; const int dimY = 2; const int dimZ = 2; std::cout << "Dimensions: x = " << dimX << ", y = " << dimY << ", z = " << dimZ << std::endl << std::endl; // Use default OpenMP value const int numThreads = -1; // use all elements in this example. const int numFrequencyElements = dimX * dimY * dimZ; // Slice length in space domain. Equivalent to dimZ for non-distributed case. const int localZLength = dimZ; // interleaved complex numbers std::vector frequencyElements; frequencyElements.reserve(2 * numFrequencyElements); // indices of frequency elements std::vector indices; indices.reserve(3 * numFrequencyElements); // initialize frequency domain values and indices double initValue = 0.0; for (int xIndex = 0; xIndex < dimX; ++xIndex) { for (int yIndex = 0; yIndex < dimY; ++yIndex) { for (int zIndex = 0; zIndex < dimZ; ++zIndex) { // init with interleaved complex numbers frequencyElements.emplace_back(initValue); frequencyElements.emplace_back(-initValue); // add index triplet for value indices.emplace_back(xIndex); indices.emplace_back(yIndex); indices.emplace_back(zIndex); initValue += 1.0; } } } std::cout << "Input:" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } // create local Grid. For distributed computations, a MPI Communicator has to be provided spfft::Grid grid(dimX, dimY, dimZ, dimX * dimY, SPFFT_PU_HOST, numThreads); // create transform spfft::Transform transform = grid.create_transform(SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, localZLength, numFrequencyElements, SPFFT_INDEX_TRIPLETS, indices.data()); // Get pointer to space domain data. Alignment fullfills requirements for std::complex. // Can also be read as std::complex elements (guaranteed by the standard to be binary compatible // since C++11). double* spaceDomain = transform.space_domain_data(SPFFT_PU_HOST); // transform backward transform.backward(frequencyElements.data(), SPFFT_PU_HOST); std::cout << std::endl << "After backward transform:" << std::endl; for (int i = 0; i < transform.local_slice_size(); ++i) { std::cout << spaceDomain[2 * i] << ", " << spaceDomain[2 * i + 1] << std::endl; } // transform forward transform.forward(SPFFT_PU_HOST, frequencyElements.data(), SPFFT_NO_SCALING); std::cout << std::endl << "After forward transform (without scaling):" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } return 0; } C - .. code-block:: c #include #include #include "spfft/spfft.h" int main(int argc, char** argv) { const int dimX = 2; const int dimY = 2; const int dimZ = 2; printf("Dimensions: x = %d, y = %d, z = %d\n\n", dimX, dimY, dimZ); /* Use default OpenMP value */ const int numThreads = -1; /* use all elements in this example. */ const int numFrequencyElements = dimX * dimY * dimZ; /* Slice length in space domain. Equivalent to dimZ for non-distributed case. */ const int localZLength = dimZ; /* interleaved complex numbers */ double* frequencyElements = (double*)malloc(2 * sizeof(double) * numFrequencyElements); /* indices of frequency elements */ int* indices = (int*)malloc(3 * sizeof(int) * numFrequencyElements); /* initialize frequency domain values and indices */ double initValue = 0.0; size_t count = 0; for (int xIndex = 0; xIndex < dimX; ++xIndex) { for (int yIndex = 0; yIndex < dimY; ++yIndex) { for (int zIndex = 0; zIndex < dimZ; ++zIndex, ++count) { /* init values */ frequencyElements[2 * count] = initValue; frequencyElements[2 * count + 1] = -initValue; /* add index triplet for value */ indices[3 * count] = xIndex; indices[3 * count + 1] = yIndex; indices[3 * count + 2] = zIndex; initValue += 1.0; } } } printf("Input:\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", frequencyElements[2 * i], frequencyElements[2 * i + 1]); } printf("\n"); SpfftError status = 0; /* create local Grid. For distributed computations, a MPI Communicator has to be provided */ SpfftGrid grid; status = spfft_grid_create(&grid, dimX, dimY, dimZ, dimX * dimY, SPFFT_PU_HOST, numThreads); if (status != SPFFT_SUCCESS) exit(status); /* create transform */ SpfftTransform transform; status = spfft_transform_create(&transform, grid, SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, localZLength, numFrequencyElements, SPFFT_INDEX_TRIPLETS, indices); if (status != SPFFT_SUCCESS) exit(status); /* grid can be safely destroyed after creating all transforms */ status = spfft_grid_destroy(grid); if (status != SPFFT_SUCCESS) exit(status); /* get pointer to space domain data. Alignment is guaranteed to fullfill requirements C complex types */ double* spaceDomain; status = spfft_transform_get_space_domain(transform, SPFFT_PU_HOST, &spaceDomain); if (status != SPFFT_SUCCESS) exit(status); /* transform backward */ status = spfft_transform_backward(transform, frequencyElements, SPFFT_PU_HOST); if (status != SPFFT_SUCCESS) exit(status); printf("After backward transform:\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", spaceDomain[2 * i], spaceDomain[2 * i + 1]); } printf("\n"); /* transform forward */ status = spfft_transform_forward(transform, SPFFT_PU_HOST, frequencyElements, SPFFT_NO_SCALING); if (status != SPFFT_SUCCESS) exit(status); printf("After forward transform (without scaling):\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", frequencyElements[2 * i], frequencyElements[2 * i + 1]); } /* destroying the final transform will free the associated memory */ status = spfft_transform_destroy(transform); if (status != SPFFT_SUCCESS) exit(status); return 0; } Fortran ------- .. code-block:: fortran program main use iso_c_binding use spfft implicit none integer :: i, j, k, counter integer, parameter :: dimX = 2 integer, parameter :: dimY = 2 integer, parameter :: dimZ = 2 integer, parameter :: maxNumLocalZColumns = dimX * dimY integer, parameter :: processingUnit = 1 integer, parameter :: maxNumThreads = -1 type(c_ptr) :: grid = c_null_ptr type(c_ptr) :: transform = c_null_ptr integer :: errorCode = 0 integer, dimension(dimX * dimY * dimZ * 3):: indices = 0 complex(C_DOUBLE_COMPLEX), dimension(dimX * dimY * dimZ):: frequencyElements complex(C_DOUBLE_COMPLEX), pointer :: spaceDomain(:,:,:) type(c_ptr) :: realValuesPtr counter = 0 do k = 1, dimZ do j = 1, dimY do i = 1, dimX frequencyElements(counter + 1) = cmplx(counter, -counter) indices(counter * 3 + 1) = i - 1 indices(counter * 3 + 2) = j - 1 indices(counter * 3 + 3) = k - 1 counter = counter + 1 end do end do end do ! print input print *, "Input:" do i = 1, size(frequencyElements) print *, frequencyElements(i) end do ! create grid and transform errorCode = spfft_grid_create(grid, dimX, dimY, dimZ, maxNumLocalZColumns, processingUnit, maxNumThreads); if (errorCode /= SPFFT_SUCCESS) error stop errorCode = spfft_transform_create(transform, grid, processingUnit, 0, dimX, dimY, dimZ, dimZ,& size(frequencyElements), SPFFT_INDEX_TRIPLETS, indices) if (errorCode /= SPFFT_SUCCESS) error stop ! grid can be safely destroyed after creating all required transforms errorCode = spfft_grid_destroy(grid) if (errorCode /= SPFFT_SUCCESS) error stop ! set space domain array to use memory allocted by the library errorCode = spfft_transform_get_space_domain(transform, processingUnit, realValuesPtr) if (errorCode /= SPFFT_SUCCESS) error stop ! transform backward errorCode = spfft_transform_backward(transform, frequencyElements, processingUnit) if (errorCode /= SPFFT_SUCCESS) error stop call c_f_pointer(realValuesPtr, spaceDomain, [dimX,dimY,dimZ]) print *, "" print *, "After backward transform:" do k = 1, size(spaceDomain, 3) do j = 1, size(spaceDomain, 2) do i = 1, size(spaceDomain, 1) print *, spaceDomain(i, j, k) end do end do end do ! transform forward (will invalidate space domain data) errorCode = spfft_transform_forward(transform, processingUnit, frequencyElements, 0) if (errorCode /= SPFFT_SUCCESS) error stop print *, "" print *, "After forward transform (without scaling):" do i = 1, size(frequencyElements) print *, frequencyElements(i) end do ! destroying the final transform will free the associated memory errorCode = spfft_transform_destroy(transform) if (errorCode /= SPFFT_SUCCESS) error stop end SpFFT-0.9.10/docs/source/exceptions.rst000066400000000000000000000001201357247005600176700ustar00rootroot00000000000000Exceptions ========== .. doxygenfile:: spfft/exceptions.hpp :project: SpFFT SpFFT-0.9.10/docs/source/grid.rst000066400000000000000000000004661357247005600164510ustar00rootroot00000000000000Grid ==== .. note:: A Grid object can be safely destroyed after transforms have been created. The transforms hold a reference counted objtect containing the allocated memory, which will remain valid until all transforms are destroyed as well. .. doxygenclass:: spfft::Grid :project: SpFFT :members: SpFFT-0.9.10/docs/source/grid_c.rst000066400000000000000000000000741357247005600167460ustar00rootroot00000000000000Grid ==== .. doxygenfile:: spfft/grid.h :project: SpFFT SpFFT-0.9.10/docs/source/grid_float.rst000066400000000000000000000007371357247005600176370ustar00rootroot00000000000000GridFloat ========= .. note:: This class is only available if single precision support is enabled, in which case the marco SPFFT_SINGLE_PRECISION is defined in config.h. .. note:: A Grid object can be safely destroyed after transforms have been created. The transforms hold a reference counted objtect containing the allocated memory, which will remain valid until all transforms are destroyed as well. .. doxygenclass:: spfft::GridFloat :project: SpFFT :members: SpFFT-0.9.10/docs/source/grid_float_c.rst000066400000000000000000000003541357247005600201340ustar00rootroot00000000000000GridFloat ========= .. note:: These functions are only available if single precision support is enabled, in which case the marco SPFFT_SINGLE_PRECISION is defined in config.h. .. doxygenfile:: spfft/grid_float.h :project: SpFFT SpFFT-0.9.10/docs/source/index.rst000066400000000000000000000051631357247005600166320ustar00rootroot00000000000000.. Copyright (c) 2019, ETH Zurich Distributed under the terms of the BSD 3-Clause License. The full license is in the file LICENSE, distributed with this software. SpFFT Documentation =================== | SpFFT - A 3D FFT library for sparse frequency domain data written in C++ with support for MPI, OpenMP, CUDA and ROCm. It was originally intended for transforms of data with spherical cutoff in frequency domain, as required by some computational material science codes. | For distributed computations, SpFFT uses a slab decomposition in space domain and pencil decomposition in frequency domain (all sparse data within a pencil must be on one rank). .. figure:: ../images/sparse_to_dense.png :align: center :width: 70% Illustration of a transform, where data on each MPI rank is identified by color. Design Goals ------------ - Sparse frequency domain input - Reuse of pre-allocated memory - Support of negative indexing for frequency domain data - Unified interface for calculations on CPUs and GPUs - Support of Complex-To-Real and Real-To-Complex transforms, where the full hermitian symmetry property is utilized. - C++, C and Fortran interfaces - Parallelization and acceleration are optional Interface Design ---------------- To allow for pre-allocation and reuse of memory, the design is based on two classes: - **Grid**: Allocates memory for transforms up to a given size in each dimension. - **Transform**: Is associated with a *Grid* and can have any size up to the *Grid* dimensions. A *Transform* holds a counted reference to the underlying *Grid*. Therefore, *Transforms* created with the same *Grid* share memory, which is only freed, once the *Grid* and all associated *Transforms* are destroyed. The user provides memory for storing sparse frequency domain data, while a *Transform* provides memory for space domain data. This implies, that executing a *Transform* will override the space domain data of all other *Transforms* associated with the same *Grid*. .. note:: The creation of Grids and Transforms, as well as the forward and backward execution may entail MPI calls and must be synchronized between all ranks. .. toctree:: :maxdepth: 2 :hidden: installation examples details .. toctree:: :maxdepth: 2 :caption: C++ API REFERENCE: :hidden: types grid grid_float transform transform_float multi_transform exceptions .. toctree:: :maxdepth: 2 :caption: C API REFERENCE: :hidden: types grid_c grid_float_c transform_c transform_float_c multi_transform_c errors_c .. Indices and tables .. ================== .. * :ref:`genindex` SpFFT-0.9.10/docs/source/installation.rst000066400000000000000000000031001357247005600202110ustar00rootroot00000000000000Installation ============ Requirements ------------ * C++ Compiler with C++11 support. Supported compilers are: * GCC 6 and later * Clang 5 and later * ICC 18.0 and later - CMake 3.11 and later - Library providing a FFTW 3.x interface (FFTW3 or Intel MKL) - For multi-threading: OpenMP support by the compiler - For compilation with GPU support: * CUDA 9.0 and later for Nvidia hardware * ROCm 2.6 and later for AMD hardware Build ----- The build system follows the standard CMake workflow. Example: .. code-block:: bash mkdir build cd build cmake .. -DSPFFT_OMP=ON -DSPFFT_MPI=ON -DSPFFT_GPU_BACKEND=CUDA -DSPFFT_SINGLE_PRECISION=OFF -DCMAKE_INSTALL_PREFIX=/usr/local make -j8 install CMake options ------------- ====================== ======= ================================================ Option Default Description ====================== ======= ================================================ SPFFT_MPI ON Enable MPI support SPFFT_OMP ON Enable multi-threading with OpenMP SPFFT_GPU_BACKEND OFF Select GPU backend. Can be OFF, CUDA or ROCM SPFFT_GPU_DIRECT OFF Use GPU aware MPI with GPUDirect SPFFT_SINGLE_PRECISION OFF Enable single precision support SPFFT_STATIC OFF Build as static library SPFFT_BUILD_TESTS OFF Build test executables for developement purposes SPFFT_INSTALL ON Add library to install target SPFFT_FORTRAN OFF Build Fortran interface module ====================== ======= ================================================ SpFFT-0.9.10/docs/source/multi_transform.rst000066400000000000000000000003601357247005600207420ustar00rootroot00000000000000Multi-Transform =============== .. note:: Only fully independent transforms can be executed in parallel. .. doxygenfile:: spfft/multi_transform.hpp :project: SpFFT .. doxygenfile:: spfft/multi_transform_float.hpp :project: SpFFT SpFFT-0.9.10/docs/source/multi_transform_c.rst000066400000000000000000000003541357247005600212470ustar00rootroot00000000000000Multi-Transform =============== .. note:: Only fully independent transforms can be executed in parallel. .. doxygenfile:: spfft/multi_transform.h :project: SpFFT .. doxygenfile:: spfft/multi_transform_float.h :project: SpFFT SpFFT-0.9.10/docs/source/transform.rst000066400000000000000000000005141357247005600175310ustar00rootroot00000000000000Transform ========= .. note:: This class only holds an internal reference counted object. The object remains in a usable state even if the associated Grid object is destroyed. In addition, copying a transform only requires an internal copy of a shared pointer. .. doxygenclass:: spfft::Transform :project: SpFFT :members: SpFFT-0.9.10/docs/source/transform_c.rst000066400000000000000000000004771357247005600200430ustar00rootroot00000000000000Transform ========= .. note:: This class only holds an internal reference counted object. The object remains in a usable state even if the associated Grid object is destroyed. In addition, copying a transform only requires an internal copy of a shared pointer. .. doxygenfile:: spfft/transform.h :project: SpFFT SpFFT-0.9.10/docs/source/transform_float.rst000066400000000000000000000007661357247005600207270ustar00rootroot00000000000000TransformFloat ============== .. note:: This class is only available if single precision support is enabled, in which case the marco SPFFT_SINGLE_PRECISION is defined in config.h. .. note:: This class only holds an internal reference counted object. The object remains in a usable state even if the associated Grid object is destroyed. In addition, copying a transform only requires an internal copy of a shared pointer. .. doxygenclass:: spfft::TransformFloat :project: SpFFT :members: SpFFT-0.9.10/docs/source/transform_float_c.rst000066400000000000000000000003731357247005600212230ustar00rootroot00000000000000TransformFloat ============== .. note:: These functions are only available if single precision support is enabled, in which case the marco SPFFT_SINGLE_PRECISION is defined in config.h. .. doxygenfile:: spfft/transform_float.h :project: SpFFT SpFFT-0.9.10/docs/source/types.rst000066400000000000000000000000771357247005600166660ustar00rootroot00000000000000Types ===== .. doxygenfile:: spfft/types.h :project: SpFFT SpFFT-0.9.10/examples/000077500000000000000000000000001357247005600143525ustar00rootroot00000000000000SpFFT-0.9.10/examples/example.c000066400000000000000000000065231357247005600161570ustar00rootroot00000000000000#include #include #include "spfft/spfft.h" int main(int argc, char** argv) { const int dimX = 2; const int dimY = 2; const int dimZ = 2; printf("Dimensions: x = %d, y = %d, z = %d\n\n", dimX, dimY, dimZ); /* Use default OpenMP value */ const int numThreads = -1; /* use all elements in this example. */ const int numFrequencyElements = dimX * dimY * dimZ; /* Slice length in space domain. Equivalent to dimZ for non-distributed case. */ const int localZLength = dimZ; /* interleaved complex numbers */ double* frequencyElements = (double*)malloc(2 * sizeof(double) * numFrequencyElements); /* indices of frequency elements */ int* indices = (int*)malloc(3 * sizeof(int) * numFrequencyElements); /* initialize frequency domain values and indices */ double initValue = 0.0; size_t count = 0; for (int xIndex = 0; xIndex < dimX; ++xIndex) { for (int yIndex = 0; yIndex < dimY; ++yIndex) { for (int zIndex = 0; zIndex < dimZ; ++zIndex, ++count) { /* init values */ frequencyElements[2 * count] = initValue; frequencyElements[2 * count + 1] = -initValue; /* add index triplet for value */ indices[3 * count] = xIndex; indices[3 * count + 1] = yIndex; indices[3 * count + 2] = zIndex; initValue += 1.0; } } } printf("Input:\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", frequencyElements[2 * i], frequencyElements[2 * i + 1]); } printf("\n"); SpfftError status = 0; /* create local Grid. For distributed computations, a MPI Communicator has to be provided */ SpfftGrid grid; status = spfft_grid_create(&grid, dimX, dimY, dimZ, dimX * dimY, SPFFT_PU_HOST, numThreads); if (status != SPFFT_SUCCESS) exit(status); /* create transform */ SpfftTransform transform; status = spfft_transform_create(&transform, grid, SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, localZLength, numFrequencyElements, SPFFT_INDEX_TRIPLETS, indices); if (status != SPFFT_SUCCESS) exit(status); /* grid can be safely destroyed after creating all transforms */ status = spfft_grid_destroy(grid); if (status != SPFFT_SUCCESS) exit(status); /* get pointer to space domain data. Alignment is guaranteed to fullfill requirements C complex types */ double* spaceDomain; status = spfft_transform_get_space_domain(transform, SPFFT_PU_HOST, &spaceDomain); if (status != SPFFT_SUCCESS) exit(status); /* transform backward */ status = spfft_transform_backward(transform, frequencyElements, SPFFT_PU_HOST); if (status != SPFFT_SUCCESS) exit(status); printf("After backward transform:\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", spaceDomain[2 * i], spaceDomain[2 * i + 1]); } printf("\n"); /* transform forward */ status = spfft_transform_forward(transform, SPFFT_PU_HOST, frequencyElements, SPFFT_NO_SCALING); if (status != SPFFT_SUCCESS) exit(status); printf("After forward transform (without scaling):\n"); for (size_t i = 0; i < dimX * dimY * dimZ; ++i) { printf("%f, %f\n", frequencyElements[2 * i], frequencyElements[2 * i + 1]); } /* destroying the final transform will free the associated memory */ status = spfft_transform_destroy(transform); if (status != SPFFT_SUCCESS) exit(status); return 0; } SpFFT-0.9.10/examples/example.cpp000066400000000000000000000055711357247005600165210ustar00rootroot00000000000000#include #include #include #include "spfft/spfft.hpp" int main(int argc, char** argv) { const int dimX = 2; const int dimY = 2; const int dimZ = 2; std::cout << "Dimensions: x = " << dimX << ", y = " << dimY << ", z = " << dimZ << std::endl << std::endl; // Use default OpenMP value const int numThreads = -1; // use all elements in this example. const int numFrequencyElements = dimX * dimY * dimZ; // Slice length in space domain. Equivalent to dimZ for non-distributed case. const int localZLength = dimZ; // interleaved complex numbers std::vector frequencyElements; frequencyElements.reserve(2 * numFrequencyElements); // indices of frequency elements std::vector indices; indices.reserve(dimX * dimY * dimZ * 3); // initialize frequency domain values and indices double initValue = 0.0; for (int xIndex = 0; xIndex < dimX; ++xIndex) { for (int yIndex = 0; yIndex < dimY; ++yIndex) { for (int zIndex = 0; zIndex < dimZ; ++zIndex) { // init with interleaved complex numbers frequencyElements.emplace_back(initValue); frequencyElements.emplace_back(-initValue); // add index triplet for value indices.emplace_back(xIndex); indices.emplace_back(yIndex); indices.emplace_back(zIndex); initValue += 1.0; } } } std::cout << "Input:" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } // create local Grid. For distributed computations, a MPI Communicator has to be provided spfft::Grid grid(dimX, dimY, dimZ, dimX * dimY, SPFFT_PU_HOST, numThreads); // create transform spfft::Transform transform = grid.create_transform(SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, localZLength, numFrequencyElements, SPFFT_INDEX_TRIPLETS, indices.data()); // Get pointer to space domain data. Alignment fullfills requirements for std::complex. // Can also be read as std::complex elements (guaranteed by the standard to be binary compatible // since C++11). double* spaceDomain = transform.space_domain_data(SPFFT_PU_HOST); // transform backward transform.backward(frequencyElements.data(), SPFFT_PU_HOST); std::cout << std::endl << "After backward transform:" << std::endl; for (int i = 0; i < transform.local_slice_size(); ++i) { std::cout << spaceDomain[2 * i] << ", " << spaceDomain[2 * i + 1] << std::endl; } // transform forward transform.forward(SPFFT_PU_HOST, frequencyElements.data(), SPFFT_NO_SCALING); std::cout << std::endl << "After forward transform (without scaling):" << std::endl; for (int i = 0; i < numFrequencyElements; ++i) { std::cout << frequencyElements[2 * i] << ", " << frequencyElements[2 * i + 1] << std::endl; } return 0; } SpFFT-0.9.10/examples/example.f90000066400000000000000000000056751357247005600163420ustar00rootroot00000000000000 program main use iso_c_binding use spfft implicit none integer :: i, j, k, counter integer, parameter :: dimX = 2 integer, parameter :: dimY = 2 integer, parameter :: dimZ = 2 integer, parameter :: maxNumLocalZColumns = dimX * dimY integer, parameter :: processingUnit = 1 integer, parameter :: maxNumThreads = -1 type(c_ptr) :: grid = c_null_ptr type(c_ptr) :: transform = c_null_ptr integer :: errorCode = 0 integer, dimension(dimX * dimY * dimZ * 3):: indices = 0 complex(C_DOUBLE_COMPLEX), dimension(dimX * dimY * dimZ):: frequencyElements complex(C_DOUBLE_COMPLEX), pointer :: spaceDomain(:,:,:) type(c_ptr) :: realValuesPtr counter = 0 do k = 1, dimZ do j = 1, dimY do i = 1, dimX frequencyElements(counter + 1) = cmplx(counter, -counter) indices(counter * 3 + 1) = i - 1 indices(counter * 3 + 2) = j - 1 indices(counter * 3 + 3) = k - 1 counter = counter + 1 end do end do end do ! print input print *, "Input:" do i = 1, size(frequencyElements) print *, frequencyElements(i) end do ! create grid and transform errorCode = spfft_grid_create(grid, dimX, dimY, dimZ, maxNumLocalZColumns, processingUnit, maxNumThreads); if (errorCode /= SPFFT_SUCCESS) error stop errorCode = spfft_transform_create(transform, grid, processingUnit, 0, dimX, dimY, dimZ, dimZ,& size(frequencyElements), SPFFT_INDEX_TRIPLETS, indices) if (errorCode /= SPFFT_SUCCESS) error stop ! grid can be safely destroyed after creating all required transforms errorCode = spfft_grid_destroy(grid) if (errorCode /= SPFFT_SUCCESS) error stop ! set space domain array to use memory allocted by the library errorCode = spfft_transform_get_space_domain(transform, processingUnit, realValuesPtr) if (errorCode /= SPFFT_SUCCESS) error stop ! transform backward errorCode = spfft_transform_backward(transform, frequencyElements, processingUnit) if (errorCode /= SPFFT_SUCCESS) error stop call c_f_pointer(realValuesPtr, spaceDomain, [dimX,dimY,dimZ]) print *, "" print *, "After backward transform:" do k = 1, size(spaceDomain, 3) do j = 1, size(spaceDomain, 2) do i = 1, size(spaceDomain, 1) print *, spaceDomain(i, j, k) end do end do end do ! transform forward (will invalidate space domain data) errorCode = spfft_transform_forward(transform, processingUnit, frequencyElements, 0) if (errorCode /= SPFFT_SUCCESS) error stop print *, "" print *, "After forward transform (without scaling):" do i = 1, size(frequencyElements) print *, frequencyElements(i) end do ! destroying the final transform will free the associated memory errorCode = spfft_transform_destroy(transform) if (errorCode /= SPFFT_SUCCESS) error stop end SpFFT-0.9.10/include/000077500000000000000000000000001357247005600141575ustar00rootroot00000000000000SpFFT-0.9.10/include/spfft/000077500000000000000000000000001357247005600153015ustar00rootroot00000000000000SpFFT-0.9.10/include/spfft/config.h.in000066400000000000000000000035521357247005600173310ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /***************** * CMAKE GENERATED *****************/ #ifndef SPFFT_CONFIG_H #define SPFFT_CONFIG_H #cmakedefine SPFFT_CUDA #cmakedefine SPFFT_ROCM #cmakedefine SPFFT_MPI #cmakedefine SPFFT_OMP #cmakedefine SPFFT_TIMING #cmakedefine SPFFT_SINGLE_PRECISION #cmakedefine SPFFT_GPU_DIRECT #include "spfft/spfft_export.h" #endif SpFFT-0.9.10/include/spfft/errors.h000066400000000000000000000070031357247005600167660ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_ERRORS_H #define SPFFT_ERRORS_H #include "spfft/config.h" enum SpfftError { /** * Success. No error. */ SPFFT_SUCCESS, /** * Unknown error. */ SPFFT_UNKNOWN_ERROR, /** * Invalid Grid or Transform handle. */ SPFFT_INVALID_HANDLE_ERROR, /** * Integer overflow. */ SPFFT_OVERFLOW_ERROR, /** * Failed to allocate memory on host. */ SPFFT_ALLOCATION_ERROR, /** * Invalid parameter. */ SPFFT_INVALID_PARAMETER_ERROR, /** * Duplicate indices given to transform. May indicate non-local z-coloumn between MPI ranks. */ SPFFT_DUPLICATE_INDICES_ERROR, /** * Invalid indices given to transform. */ SPFFT_INVALID_INDICES_ERROR, /** * Library not compiled with MPI support. */ SPFFT_MPI_SUPPORT_ERROR, /** * MPI error. Only returned if error code of MPI API calls is non-zero. */ SPFFT_MPI_ERROR, /** * Parameters differ between MPI ranks. */ SPFFT_MPI_PARAMETER_MISMATCH_ERROR, /** * Failed execution on host. */ SPFFT_HOST_EXECUTION_ERROR, /** * FFTW library error. */ SPFFT_FFTW_ERROR, /** * Generic GPU error. */ SPFFT_GPU_ERROR, /** * Detected error on GPU from previous GPU API / kernel calls. */ SPFFT_GPU_PRECEDING_ERROR, /** * Library not compiled with GPU support. */ SPFFT_GPU_SUPPORT_ERROR, /** * Failed allocation on GPU. */ SPFFT_GPU_ALLOCATION_ERROR, /** * Failed to launch kernel on GPU. */ SPFFT_GPU_LAUNCH_ERROR, /** * No GPU device detected. */ SPFFT_GPU_NO_DEVICE_ERROR, /** * Invalid value passed to GPU API. */ SPFFT_GPU_INVALID_VALUE_ERROR, /** * Invalid device pointer used. */ SPFFT_GPU_INVALID_DEVICE_PTR_ERROR, /** * Failed to copy from / to GPU. */ SPFFT_GPU_COPY_ERROR, /** * Failure in GPU FFT library call. */ SPFFT_GPU_FFT_ERROR }; #ifndef __cplusplus /*! \cond PRIVATE */ // C only typedef enum SpfftError SpfftError; /*! \endcond */ #endif // cpp #endif SpFFT-0.9.10/include/spfft/exceptions.hpp000066400000000000000000000215341357247005600202000ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_EXCEPTIONS_H #define SPFFT_EXCEPTIONS_H #include #include "spfft/config.h" #include "spfft/errors.h" namespace spfft { /** * A generic error. Base type for all other exceptions. */ class SPFFT_EXPORT GenericError : public std::exception { public: auto what() const noexcept -> const char* override { return "SpFFT: Generic error"; } virtual auto error_code() const noexcept -> SpfftError { return SpfftError::SPFFT_UNKNOWN_ERROR; } }; /** * Overflow of integer values. */ class SPFFT_EXPORT OverflowError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Overflow error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_OVERFLOW_ERROR; } }; /** * Failed allocation on host. */ class SPFFT_EXPORT HostAllocationError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Host allocation error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_ALLOCATION_ERROR; } }; /** * Invalid parameter. */ class SPFFT_EXPORT InvalidParameterError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Invalid parameter error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_INVALID_PARAMETER_ERROR; } }; /** * Duplicate indices given to transform. May indicate non-local z-coloumn between MPI ranks. */ class SPFFT_EXPORT DuplicateIndicesError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Duplicate indices error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_DUPLICATE_INDICES_ERROR; } }; /** * Invalid indices given to transform. */ class SPFFT_EXPORT InvalidIndicesError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Invalid indices error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_INVALID_INDICES_ERROR; } }; /** * Library not compiled with MPI support. */ class SPFFT_EXPORT MPISupportError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Not compiled with MPI support error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_MPI_SUPPORT_ERROR; } }; /** * MPI error. Only thrown if error code of MPI API calls is non-zero. */ class SPFFT_EXPORT MPIError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: MPI error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_MPI_ERROR; } }; /** * Parameters differ between MPI ranks. */ class SPFFT_EXPORT MPIParameterMismatchError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Mismatched parameters between MPI ranks"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_MPI_PARAMETER_MISMATCH_ERROR; } }; /** * Failed execution on host. */ class SPFFT_EXPORT HostExecutionError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Host execution error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_HOST_EXECUTION_ERROR; } }; /** * FFTW library error. */ class SPFFT_EXPORT FFTWError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: FFTW error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_FFTW_ERROR; } }; /** * Unknown internal error. */ class SPFFT_EXPORT InternalError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: Internal error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_FFTW_ERROR; } }; // ================================== // GPU Errors // ================================== /** * Generic GPU error. Base type for all GPU related exceptions. */ class SPFFT_EXPORT GPUError : public GenericError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_ERROR; } }; /** * Library not compiled with GPU support. */ class SPFFT_EXPORT GPUSupportError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: Not compiled with GPU support"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_SUPPORT_ERROR; } }; /** * Detected error on GPU from previous GPU API / kernel calls. */ class SPFFT_EXPORT GPUPrecedingError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: Detected error from preceding gpu calls."; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_PRECEDING_ERROR; } }; /** * Failed allocation on GPU. */ class SPFFT_EXPORT GPUAllocationError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU allocation error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_ALLOCATION_ERROR; } }; /** * Failed to launch kernel on GPU. */ class SPFFT_EXPORT GPULaunchError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU launch error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_LAUNCH_ERROR; } }; /** * No GPU device detected. */ class SPFFT_EXPORT GPUNoDeviceError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: no GPU available"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_NO_DEVICE_ERROR; } }; /** * Invalid value passed to GPU API. */ class SPFFT_EXPORT GPUInvalidValueError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU call with invalid value"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_INVALID_VALUE_ERROR; } }; /** * Invalid device pointer used. */ class SPFFT_EXPORT GPUInvalidDevicePointerError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: Invalid GPU pointer"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_INVALID_DEVICE_PTR_ERROR; } }; /** * Failed to copy from / to GPU. */ class SPFFT_EXPORT GPUCopyError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU Memory copy error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_COPY_ERROR; } }; /** * Failure in GPU FFT library call. */ class SPFFT_EXPORT GPUFFTError : public GPUError { public: auto what() const noexcept -> const char* override { return "SpFFT: GPU FFT error"; } auto error_code() const noexcept -> SpfftError override { return SpfftError::SPFFT_GPU_FFT_ERROR; } }; } // namespace spfft #endif SpFFT-0.9.10/include/spfft/grid.h000066400000000000000000000165511357247005600164070ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GRID_H #define SPFFT_GRID_H #include "spfft/config.h" #include "spfft/errors.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif #ifdef __cplusplus extern "C" { #endif /** * Grid handle. */ typedef void* SpfftGrid; /** * Constructor for a local grid. * * @param[out] grid Handle to grid. * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_create(SpfftGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads); #ifdef SPFFT_MPI /** * Constructor for a distributed grid. * * @param[out] grid Handle to grid. * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the * local MPI rank. * @param[in] maxLocalZLength Maximum length in z in space domain for the local MPI rank. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @param[in] comm The MPI communicator to use. Will be duplicated for internal use. * @param[in] exchangeType The type of MPI exchange to use. Possible values are * SPFFT_EXCH_DEFAULT, SPFFT_EXCH_BUFFERED, SPFFT_EXCH_COMPACT_BUFFERED and SPFFT_EXCH_UNBUFFERED. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_create_distributed(SpfftGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType); #endif /** * Destroy a grid. * * A grid can be safely destroyed independet from any related transforms. The internal memory * is released, once all associated transforms are destroyed as well (through internal reference * counting). * * @param[in] grid Handle to grid. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_destroy(SpfftGrid grid); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimX Maximum dimension in x. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_max_dim_x(SpfftGrid grid, int* dimX); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimY Maximum dimension in y. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_max_dim_y(SpfftGrid grid, int* dimY); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimZ Maximum dimension in z. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_max_dim_z(SpfftGrid grid, int* dimZ); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the local MPI * rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_max_num_local_z_columns(SpfftGrid grid, int* maxNumLocalZColumns); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] maxLocalZLength Maximum length in z in space domain of the local MPI rank. * rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_max_local_z_length(SpfftGrid grid, int* maxLocalZLength); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] processingUnit The processing unit, the grid has prepared for. Can be SPFFT_PU_HOST * or SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_processing_unit(SpfftGrid grid, SpfftProcessingUnitType* processingUnit); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] deviceId The GPU device id used. Returns always 0, if no GPU support is enabled. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_device_id(SpfftGrid grid, int* deviceId); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] numThreads The exact number of threads used by transforms created from this grid. May * be less than the maximum given to the constructor. Always 1, if not compiled with OpenMP support. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_num_threads(SpfftGrid grid, int* numThreads); #ifdef SPFFT_MPI /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] comm The internal MPI communicator. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_grid_communicator(SpfftGrid grid, MPI_Comm* comm); #endif #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/grid.hpp000066400000000000000000000170541357247005600167460ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GRID_HPP #define SPFFT_GRID_HPP #include #include "spfft/config.h" #include "spfft/transform.hpp" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif namespace spfft { // Forward declaration for internal use template class SPFFT_NO_EXPORT GridInternal; /** * A Grid, which provides pre-allocated memory for double precision transforms. */ class SPFFT_EXPORT Grid { public: /** * Constructor for a local grid. * * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ Grid(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads); #ifdef SPFFT_MPI /** * Constructor for a distributed grid. * * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the * local MPI rank. * @param[in] maxLocalZLength Maximum length in z in space domain for the local MPI rank. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @param[in] comm The MPI communicator to use. Will be duplicated for internal use. * @param[in] exchangeType The type of MPI exchange to use. Possible values are * SPFFT_EXCH_DEFAULT, SPFFT_EXCH_BUFFERED, SPFFT_EXCH_COMPACT_BUFFERED and SPFFT_EXCH_UNBUFFERED. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ Grid(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType); #endif /** * Custom copy constructor. * * Creates a independent copy. Calls MPI functions for the distributed case. */ Grid(const Grid&); /** * Default move constructor. */ Grid(Grid&&) = default; /** * Custom copy operator. * * Creates a independent copy. Calls MPI functions for the distributed case. */ Grid& operator=(const Grid&); /** * Default move operator. */ Grid& operator=(Grid&&) = default; /** * Creates a transform from this grid object. * * @param[in] processingUnit The processing unit type to use. Must be either SPFFT_PU_HOST or * SPFFT_PU_GPU and be supported by the grid itself. * @param[in] transformType The transform type (complex to complex or real to complex). Can be * SPFFT_TRANS_C2C or SPFFT_TRANS_R2C. * @param[in] dimX The dimension in x. The maximum allowed depends on the grid parameters. * @param[in] dimY The dimension in y. The maximum allowed depends on the grid parameters. * @param[in] dimZ The dimension in z. The maximum allowed depends on the grid parameters. * @param[in] localZLength The length in z in space domain of the local MPI rank. * @param[in] numLocalElements The number of elements in frequency domain of the local MPI * rank. * @param[in] indexFormat The index format. Only SPFFT_INDEX_TRIPLETS currently supported. * @param[in] indices Pointer to the frequency indices. Posive and negative indexing is supported. * @return Transform * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ Transform create_transform(SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) const; /** * Access a grid parameter. * @return Maximum dimension in x. */ int max_dim_x() const; /** * Access a grid parameter. * @return Maximum dimension in y. */ int max_dim_y() const; /** * Access a grid parameter. * @return Maximum dimension in z. */ int max_dim_z() const; /** * Access a grid parameter. * @return Maximum number of z-columns in frequency domain of the local MPI rank. */ int max_num_local_z_columns() const; /** * Access a grid parameter. * @return Maximum length in z in space domain of the local MPI rank. */ int max_local_z_length() const; /** * Access a grid parameter. * @return The processing unit, the grid has prepared for. Can be SPFFT_PU_HOST or SPFFT_PU_GPU or * SPFFT_PU_HOST | SPFFT_PU_GPU. */ SpfftProcessingUnitType processing_unit() const; /** * Access a grid parameter. * @return The GPU device id used. Always returns 0, if no GPU support is enabled. */ int device_id() const; /** * Access a grid parameter. * @return The exact number of threads used by transforms created from this grid. May be less than * the maximum given to the constructor. Always 1, if not compiled with OpenMP support. */ int num_threads() const; #ifdef SPFFT_MPI /** * Access a grid parameter. * @return The internal MPI communicator. */ MPI_Comm communicator() const; #endif private: std::shared_ptr> grid_; }; } // namespace spfft #endif SpFFT-0.9.10/include/spfft/grid_float.h000066400000000000000000000165711357247005600175760ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GRID_FLOAT_H #define SPFFT_GRID_FLOAT_H #include "spfft/config.h" #include "spfft/errors.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif #ifdef __cplusplus extern "C" { #endif /** * Grid handle. */ typedef void* SpfftFloatGrid; /** * Constructor for a single precision local grid. * * @param[out] grid Handle to grid. * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_create(SpfftFloatGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads); #ifdef SPFFT_MPI /** * Constructor for a single precision distributed grid. * * @param[out] grid Handle to grid. * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the * local MPI rank. * @param[in] maxLocalZLength Maximum length in z in space domain for the local MPI rank. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @param[in] comm The MPI communicator to use. Will be duplicated for internal use. * @param[in] exchangeType The type of MPI exchange to use. Possible values are * SPFFT_EXCH_DEFAULT, SPFFT_EXCH_BUFFERED, SPFFT_EXCH_COMPACT_BUFFERED and SPFFT_EXCH_UNBUFFERED. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_create_distributed( SpfftFloatGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType); #endif /** * Destroy a grid. * * A grid can be safely destroyed independet from any related transforms. The internal memory * is released, once all associated transforms are destroyed as well (through internal reference * counting). * * @param[in] grid Handle to grid. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_destroy(SpfftFloatGrid grid); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimX Maximum dimension in x. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_max_dim_x(SpfftFloatGrid grid, int* dimX); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimY Maximum dimension in y. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_max_dim_y(SpfftFloatGrid grid, int* dimY); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] dimZ Maximum dimension in z. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_max_dim_z(SpfftFloatGrid grid, int* dimZ); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the local MPI * rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_max_num_local_z_columns(SpfftFloatGrid grid, int* maxNumLocalZColumns); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] maxLocalZLength Maximum length in z in space domain of the local MPI rank. * rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_max_local_z_length(SpfftFloatGrid grid, int* maxLocalZLength); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] processingUnit The processing unit, the grid has prepared for. Can be SPFFT_PU_HOST * or SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_processing_unit(SpfftFloatGrid grid, SpfftProcessingUnitType* processingUnit); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] deviceId The GPU device id used. Returns always 0, if no GPU support is enabled. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_device_id(SpfftFloatGrid grid, int* deviceId); /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] numThreads The exact number of threads used by transforms created from this grid. May * be less than the maximum given to the constructor. Always 1, if not compiled with OpenMP support. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_num_threads(SpfftFloatGrid grid, int* numThreads); #ifdef SPFFT_MPI /** * Access a grid parameter. * @param[in] grid Handle to grid. * @param[out] comm The internal MPI communicator. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_grid_communicator(SpfftFloatGrid grid, MPI_Comm* comm); #endif #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/grid_float.hpp000066400000000000000000000172171357247005600201340ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GRID_FLOAT_HPP #define SPFFT_GRID_FLOAT_HPP #include #include "spfft/config.h" #include "spfft/transform_float.hpp" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif namespace spfft { // Forward declaration for internal use template class SPFFT_NO_EXPORT GridInternal; #ifdef SPFFT_SINGLE_PRECISION /** * A Grid, which provides pre-allocated memory for single precision transforms. */ class SPFFT_EXPORT GridFloat { public: /** * Constructor for a local grid. * * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ GridFloat(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads); #ifdef SPFFT_MPI /** * Constructor for a distributed grid. * * @param[in] maxDimX Maximum dimension in x. * @param[in] maxDimY Maximum dimension in y. * @param[in] maxDimZ Maximum dimension in z. * @param[in] maxNumLocalZColumns Maximum number of z-columns in frequency domain of the * local MPI rank. * @param[in] maxLocalZLength Maximum length in z in space domain for the local MPI rank. * @param[in] processingUnit The processing unit type to prepare for. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU or SPFFT_PU_HOST | SPFFT_PU_GPU. * @param[in] maxNumThreads The maximum number of threads, transforms created with this grid are * allowed to use. If smaller than 1, the OpenMP default value is used. * @param[in] comm The MPI communicator to use. Will be duplicated for internal use. * @param[in] exchangeType The type of MPI exchange to use. Possible values are * SPFFT_EXCH_DEFAULT, SPFFT_EXCH_BUFFERED, SPFFT_EXCH_COMPACT_BUFFERED and SPFFT_EXCH_UNBUFFERED. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ GridFloat(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType); #endif /** * Custom copy constructor. * * Creates a independent copy. Calls MPI functions for the distributed case. */ GridFloat(const GridFloat&); /** * Default move constructor. */ GridFloat(GridFloat&&) = default; /** * Custom copy operator. * * Creates a independent copy. Calls MPI functions for the distributed case. */ GridFloat& operator=(const GridFloat&); /** * Default move operator. */ GridFloat& operator=(GridFloat&&) = default; /** * Creates a transform from this grid object. * * @param[in] processingUnit The processing unit type to use. Must be either SPFFT_PU_HOST or * SPFFT_PU_GPU and be supported by the grid itself. * @param[in] transformType The transform type (complex to complex or real to complex). Can be * SPFFT_TRANS_C2C or SPFFT_TRANS_R2C. * @param[in] dimX The dimension in x. The maximum allowed depends on the grid parameters. * @param[in] dimY The dimension in y. The maximum allowed depends on the grid parameters. * @param[in] dimZ The dimension in z. The maximum allowed depends on the grid parameters. * @param[in] localZLength The length in z in space domain of the local MPI rank. * @param[in] numLocalElements The number of elements in frequency domain of the local MPI * rank. * @param[in] indexFormat The index format. Only SPFFT_INDEX_TRIPLETS currently supported. * @param[in] indices Pointer to the frequency indices. Posive and negative indexing is supported. * @return Transform * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ TransformFloat create_transform(SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) const; /** * Access a grid parameter. * @return Maximum dimension in x. */ int max_dim_x() const; /** * Access a grid parameter. * @return Maximum dimension in y. */ int max_dim_y() const; /** * Access a grid parameter. * @return Maximum dimension in z. */ int max_dim_z() const; /** * Access a grid parameter. * @return Maximum number of z-columns in frequency domain of the local MPI rank. */ int max_num_local_z_columns() const; /** * Access a grid parameter. * @return Maximum length in z in space domain of the local MPI rank. */ int max_local_z_length() const; /** * Access a grid parameter. * @return The processing unit, the grid has prepared for. Can be SPFFT_PU_HOST or SPFFT_PU_GPU or * SPFFT_PU_HOST | SPFFT_PU_GPU. */ SpfftProcessingUnitType processing_unit() const; /** * Access a grid parameter. * @return The GPU device id used. Always returns 0, if no GPU support is enabled. */ int device_id() const; /** * Access a grid parameter. * @return The exact number of threads used by transforms created from this grid. May be less than * the maximum given to the constructor. Always 1, if not compiled with OpenMP support. */ int num_threads() const; #ifdef SPFFT_MPI MPI_Comm communicator() const; #endif private: /*! \cond PRIVATE */ std::shared_ptr> grid_; /*! \endcond */ }; #endif } // namespace spfft #endif SpFFT-0.9.10/include/spfft/multi_transform.h000066400000000000000000000063351357247005600207060ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MULTI_TRANSFORM_H #define SPFFT_MULTI_TRANSFORM_H #include "spfft/config.h" #include "spfft/transform.h" #include "spfft/types.h" #ifdef __cplusplus extern "C" { #endif /** * Execute multiple independent forward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputLocations Input locations for each transform. * @param[out] outputPointers Output pointers for each transform. * @param[in] scalingTypes Scaling types for each transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_multi_transform_forward(int numTransforms, SpfftTransform* transforms, SpfftProcessingUnitType* inputLocations, double** outputPointers, SpfftScalingType* scalingTypes); /** * Execute multiple independent backward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputPointers Input pointers for each transform. * @param[in] outputLocations Output locations for each transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_multi_transform_backward(int numTransforms, SpfftTransform* transforms, double** inputPointers, SpfftProcessingUnitType* outputLocations); #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/multi_transform.hpp000066400000000000000000000063251357247005600212450ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MULTI_TRANSFORM_HPP #define SPFFT_MULTI_TRANSFORM_HPP #include "spfft/config.h" #include "spfft/transform.hpp" #include "spfft/types.h" namespace spfft { /** * Execute multiple independent forward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputLocations Input locations for each transform. * @param[out] outputPointers Output pointers for each transform. * @param[in] scalingTypes Scaling types for each transform. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ SPFFT_EXPORT void multi_transform_forward(int numTransforms, Transform* transforms, SpfftProcessingUnitType* inputLocations, double** outputPointers, SpfftScalingType* scalingTypes); /** * Execute multiple independent backward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputPointers Input pointers for each transform. * @param[in] outputLocations Output locations for each transform. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ SPFFT_EXPORT void multi_transform_backward(int numTransforms, Transform* transforms, double** inputPointers, SpfftProcessingUnitType* outputLocations); } // namespace spfft #endif SpFFT-0.9.10/include/spfft/multi_transform_float.h000066400000000000000000000062651357247005600220750ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MULTI_TRANSFORM_FLOAT_H #define SPFFT_MULTI_TRANSFORM_FLOAT_H #include "spfft/config.h" #include "spfft/transform_float.h" #include "spfft/types.h" #ifdef __cplusplus extern "C" { #endif /** * Execute multiple independent forward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputLocations Input locations for each transform. * @param[out] outputPointers Output pointers for each transform. * @param[in] scalingTypes Scaling types for each transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_multi_transform_forward(int numTransforms, SpfftFloatTransform* transforms, SpfftProcessingUnitType* inputLocations, float** outputPointers, SpfftScalingType* scalingTypes); /** * Execute multiple independent backward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputPointers Input pointers for each transform. * @param[in] outputLocations Output locations for each transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_multi_transform_backward( int numTransforms, SpfftFloatTransform* transforms, float** inputPointers, SpfftProcessingUnitType* outputLocations); #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/multi_transform_float.hpp000066400000000000000000000064101357247005600224250ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MULTI_TRANSFORM_HPP #define SPFFT_MULTI_TRANSFORM_HPP #include "spfft/config.h" #include "spfft/transform_float.hpp" #include "spfft/types.h" namespace spfft { #ifdef SPFFT_SINGLE_PRECISION /** * Execute multiple independent forward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputLocations Input locations for each transform. * @param[out] outputPointers Output pointers for each transform. * @param[in] scalingTypes Scaling types for each transform. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ SPFFT_EXPORT void multi_transform_forward(int numTransforms, TransformFloat* transforms, SpfftProcessingUnitType* inputLocations, float** outputPointers, SpfftScalingType* scalingTypes); /** * Execute multiple independent backward transforms at once by internal pipelining. * * @param[in] numTransforms Number of transforms to execute. * @param[in] transforms Transforms to execute. * @param[in] inputPointers Input pointers for each transform. * @param[in] outputLocations Output locations for each transform. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ SPFFT_EXPORT void multi_transform_backward(int numTransforms, TransformFloat* transforms, float** inputPointers, SpfftProcessingUnitType* outputLocations); #endif } // namespace spfft #endif SpFFT-0.9.10/include/spfft/spfft.f90000066400000000000000000000476541357247005600167630ustar00rootroot00000000000000 ! Copyright (c) 2019 ETH Zurich, Simon Frasch ! ! Redistribution and use in source and binary forms, with or without ! modification, are permitted provided that the following conditions are met: ! ! 1. Redistributions of source code must retain the above copyright notice, ! this list of conditions and the following disclaimer. ! 2. Redistributions in binary form must reproduce the above copyright ! notice, this list of conditions and the following disclaimer in the ! documentation and/or other materials provided with the distribution. ! 3. Neither the name of the copyright holder nor the names of its contributors ! may be used to endorse or promote products derived from this software ! without specific prior written permission. ! ! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ! AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ! IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ! ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE ! LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ! CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF ! SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS ! INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN ! CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ! ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ! POSSIBILITY OF SUCH DAMAGE. module spfft use iso_c_binding implicit none ! Constants integer(c_int), parameter :: & SPFFT_EXCH_DEFAULT = 0, & SPFFT_EXCH_BUFFERED = 1, & SPFFT_EXCH_BUFFERED_FLOAT = 2, & SPFFT_EXCH_COMPACT_BUFFERED = 3, & SPFFT_EXCH_COMPACT_BUFFERED_FLOAT = 4, & SPFFT_EXCH_UNBUFFERED = 5, & SPFFT_PU_HOST = 1, & SPFFT_PU_GPU = 2, & SPFFT_INDEX_TRIPLETS = 0, & SPFFT_TRANS_C2C = 0, & SPFFT_TRANS_R2C = 1, & SPFFT_NO_SCALING = 0, & SPFFT_FULL_SCALING = 1, & SPFFT_SUCCESS = 0, & SPFFT_UNKNOWN_ERROR = 1, & SPFFT_INVALID_HANDLE_ERROR = 2, & SPFFT_OVERFLOW_ERROR = 3, & SPFFT_ALLOCATION_ERROR = 4, & SPFFT_INVALID_PARAMETER_ERROR = 5, & SPFFT_DUPLICATE_INDICES_ERROR = 6, & SPFFT_INVALID_INDICES_ERROR = 7, & SPFFT_MPI_SUPPORT_ERROR = 8, & SPFFT_MPI_ERROR = 9, & SPFFT_MPI_PARAMETER_MISMATCH_ERROR = 10, & SPFFT_HOST_EXECUTION_ERROR = 11, & SPFFT_FFTW_ERROR = 12, & SPFFT_GPU_ERROR = 13, & SPFFT_GPU_PRECEDING_ERROR = 14, & SPFFT_GPU_SUPPORT_ERROR = 15, & SPFFT_GPU_ALLOCATION_ERROR = 16, & SPFFT_GPU_LAUNCH_ERROR = 17, & SPFFT_GPU_NO_DEVICE_ERROR = 18, & SPFFT_GPU_INVALID_VALUE_ERROR = 19, & SPFFT_GPU_INVALID_DEVICE_PTR_ERROR = 20, & SPFFT_GPU_COPY_ERROR = 21, & SPFFT_GPU_FFT_ERROR = 22 interface !-------------------------- ! Grid !-------------------------- integer(c_int) function spfft_grid_create(grid, maxDimX, maxDimY, maxDimZ, & maxNumLocalZColumns, processingUnit, maxNumThreads) bind(C) use iso_c_binding type(c_ptr), intent(out) :: grid integer(c_int), value :: maxDimX integer(c_int), value :: maxDimY integer(c_int), value :: maxDimZ integer(c_int), value :: maxNumLocalZColumns integer(c_int), value :: processingUnit integer(c_int), value :: maxNumThreads end function integer(c_int) function spfft_grid_create_distributed(grid, maxDimX, maxDimY, maxDimZ, & maxNumLocalZColumns, maxLocalZLength, processingUnit, maxNumThreads,& comm, exchangeType) bind(C, name='spfft_grid_create_distributed_fortran') use iso_c_binding type(c_ptr), intent(out) :: grid integer(c_int), value :: maxDimX integer(c_int), value :: maxDimY integer(c_int), value :: maxDimZ integer(c_int), value :: maxNumLocalZColumns integer(c_int), value :: maxLocalZLength integer(c_int), value :: processingUnit integer(c_int), value :: maxNumThreads integer(c_int), value :: comm integer(c_int), value :: exchangeType end function integer(c_int) function spfft_grid_destroy(grid) bind(C) use iso_c_binding type(c_ptr), value :: grid end function integer(c_int) function spfft_grid_max_dim_x(grid, dimX) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimX end function integer(c_int) function spfft_grid_max_dim_y(grid, dimY) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimY end function integer(c_int) function spfft_grid_max_dim_z(grid, dimZ) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimZ end function integer(c_int) function spfft_grid_max_num_local_z_columns(grid, maxNumLocalZColumns) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: maxNumLocalZColumns end function integer(c_int) function spfft_grid_max_local_z_length(grid, maxLocalZLength) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: maxLocalZLength end function integer(c_int) function spfft_grid_processing_unit(grid, processingUnit) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: processingUnit end function integer(c_int) function spfft_grid_device_id(grid, deviceId) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: deviceId end function integer(c_int) function spfft_grid_num_threads(grid, numThreads) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: numThreads end function integer(c_int) function spfft_grid_communicator(grid, comm) & bind(C, name="spfft_grid_communicator_fortran") use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: comm end function !-------------------------- ! Grid Float !-------------------------- integer(c_int) function spfft_float_grid_create(grid, maxDimX, maxDimY, maxDimZ, & maxNumLocalZColumns, processingUnit, maxNumThreads) bind(C) use iso_c_binding type(c_ptr), intent(out) :: grid integer(c_int), value :: maxDimX integer(c_int), value :: maxDimY integer(c_int), value :: maxDimZ integer(c_int), value :: maxNumLocalZColumns integer(c_int), value :: processingUnit integer(c_int), value :: maxNumThreads end function integer(c_int) function spfft_float_grid_create_distributed(grid, maxDimX, maxDimY, maxDimZ, & maxNumLocalZColumns, maxLocalZLength, processingUnit, maxNumThreads,& comm, exchangeType) bind(C, name='spfft_float_grid_create_distributed_fortran') use iso_c_binding type(c_ptr), intent(out) :: grid integer(c_int), value :: maxDimX integer(c_int), value :: maxDimY integer(c_int), value :: maxDimZ integer(c_int), value :: maxNumLocalZColumns integer(c_int), value :: maxLocalZLength integer(c_int), value :: processingUnit integer(c_int), value :: maxNumThreads integer(c_int), value :: comm integer(c_int), value :: exchangeType end function integer(c_int) function spfft_float_grid_destroy(grid) bind(C) use iso_c_binding type(c_ptr), value :: grid end function integer(c_int) function spfft_float_grid_max_dim_x(grid, dimX) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimX end function integer(c_int) function spfft_float_grid_max_dim_y(grid, dimY) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimY end function integer(c_int) function spfft_float_grid_max_dim_z(grid, dimZ) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: dimZ end function integer(c_int) function spfft_float_grid_max_num_local_z_columns(grid, maxNumLocalZColumns) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: maxNumLocalZColumns end function integer(c_int) function spfft_float_grid_max_local_z_length(grid, maxLocalZLength) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: maxLocalZLength end function integer(c_int) function spfft_float_grid_processing_unit(grid, processingUnit) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: processingUnit end function integer(c_int) function spfft_float_grid_device_id(grid, deviceId) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: deviceId end function integer(c_int) function spfft_float_grid_num_threads(grid, numThreads) bind(C) use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: numThreads end function integer(c_int) function spfft_float_grid_communicator(grid, comm) & bind(C, name="spfft_float_grid_communicator_fortran") use iso_c_binding type(c_ptr), value :: grid integer(c_int), intent(out) :: comm end function !-------------------------- ! Transform !-------------------------- integer(c_int) function spfft_transform_create(transform, grid, processingUnit, & transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices) bind(C) use iso_c_binding type(c_ptr), intent(out) :: transform type(c_ptr), value :: grid integer(c_int), value :: processingUnit integer(c_int), value :: transformType integer(c_int), value :: dimX integer(c_int), value :: dimY integer(c_int), value :: dimZ integer(c_int), value :: localZLength integer(c_int), value :: numLocalElements integer(c_int), value :: indexFormat integer(c_int), dimension(*), intent(in) :: indices end function integer(c_int) function spfft_transform_destroy(transform) bind(C) use iso_c_binding type(c_ptr), value :: transform end function integer(c_int) function spfft_transform_clone(transform, newTransform) bind(C) use iso_c_binding type(c_ptr), value :: transform type(c_ptr), intent(out) :: newTransform end function integer(c_int) function spfft_transform_backward(transform, input, & outputLocation) bind(C) use iso_c_binding type(c_ptr), value :: transform complex(c_double), dimension(*), intent(in) :: input integer(c_int), value :: outputLocation end function integer(c_int) function spfft_transform_forward(transform, inputLocation, & output, scaling) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), value :: inputLocation complex(c_double), dimension(*), intent(out) :: output integer(c_int), value :: scaling end function integer(c_int) function spfft_transform_get_space_domain(transform, & dataLocation, dataPtr) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), value :: dataLocation type(c_ptr), intent(out) :: dataPtr end function integer(c_int) function spfft_transform_dim_x(transform, dimX) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimX end function integer(c_int) function spfft_transform_dim_y(transform, dimY) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimY end function integer(c_int) function spfft_transform_dim_z(transform, dimZ) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimZ end function integer(c_int) function spfft_transform_local_z_length(transform, localZLength) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: localZLength end function integer(c_int) function spfft_transform_local_slice_size(transform, size) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: size end function integer(c_int) function spfft_transform_local_z_offset(transform, offset) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: offset end function integer(c_int) function spfft_transform_global_size(transform, globalSize) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_long_long), intent(out) :: globalSize end function integer(c_int) function spfft_transform_num_local_elements(transform, numLocalElements) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: numLocalElements end function integer(c_int) function spfft_transform_num_global_elements(transform, numGlobalElements) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_long_long), intent(out) :: numGlobalElements end function integer(c_int) function spfft_transform_device_id(transform, deviceId) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: deviceId end function integer(c_int) function spfft_transform_num_threads(transform, numThreads) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: numThreads end function integer(c_int) function spfft_transform_communicator(transform, comm) & bind(C, name="spfft_transform_communicator_fortran") use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: comm end function integer(c_int) function spfft_multi_transform_forward(numTransforms, transforms,& inputLocations, outputPointers, scalingTypes) bind(C) use iso_c_binding integer(c_int), value :: numTransforms type(c_ptr), value :: transforms type(c_ptr), value :: inputLocations type(c_ptr), value :: outputPointers type(c_ptr), value :: scalingTypes end function integer(c_int) function spfft_multi_transform_backward(numTransforms, transforms,& inputPointers, outputLocations) bind(C) use iso_c_binding integer(c_int), value :: numTransforms type(c_ptr), value :: transforms type(c_ptr), value :: inputPointers type(c_ptr), value :: outputLocations end function !-------------------------- ! Transform Float !-------------------------- integer(c_int) function spfft_float_transform_create(transform, grid, processingUnit, & transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices) bind(C) use iso_c_binding type(c_ptr), intent(out) :: transform type(c_ptr), value :: grid integer(c_int), value :: processingUnit integer(c_int), value :: transformType integer(c_int), value :: dimX integer(c_int), value :: dimY integer(c_int), value :: dimZ integer(c_int), value :: localZLength integer(c_int), value :: numLocalElements integer(c_int), value :: indexFormat integer(c_int), dimension(*), intent(in) :: indices end function integer(c_int) function spfft_float_transform_destroy(transform) bind(C) use iso_c_binding type(c_ptr), value :: transform end function integer(c_int) function spfft_float_transform_clone(transform, newTransform) bind(C) use iso_c_binding type(c_ptr), value :: transform type(c_ptr), intent(out) :: newTransform end function integer(c_int) function spfft_float_transform_backward(transform, input, & outputLocation) bind(C) use iso_c_binding type(c_ptr), value :: transform complex(c_double), dimension(*), intent(in) :: input integer(c_int), value :: outputLocation end function integer(c_int) function spfft_float_transform_forward(transform, inputLocation, & output, scaling) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), value :: inputLocation complex(c_double), dimension(*), intent(out) :: output integer(c_int), value :: scaling end function integer(c_int) function spfft_float_transform_get_space_domain(transform, & dataLocation, dataPtr) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), value :: dataLocation type(c_ptr), intent(out) :: dataPtr end function integer(c_int) function spfft_float_transform_dim_x(transform, dimX) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimX end function integer(c_int) function spfft_float_transform_dim_y(transform, dimY) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimY end function integer(c_int) function spfft_float_transform_dim_z(transform, dimZ) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: dimZ end function integer(c_int) function spfft_float_transform_local_z_length(transform, localZLength) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: localZLength end function integer(c_int) function spfft_float_transform_local_slice_size(transform, size) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: size end function integer(c_int) function spfft_float_transform_local_z_offset(transform, offset) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: offset end function integer(c_int) function spfft_float_transform_global_size(transform, globalSize) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_long_long), intent(out) :: globalSize end function integer(c_int) function spfft_float_transform_num_local_elements(transform, numLocalElements) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: numLocalElements end function integer(c_int) function spfft_float_transform_num_global_elements(transform, numGlobalElements) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_long_long), intent(out) :: numGlobalElements end function integer(c_int) function spfft_float_transform_device_id(transform, deviceId) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: deviceId end function integer(c_int) function spfft_float_transform_num_threads(transform, numThreads) bind(C) use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: numThreads end function integer(c_int) function spfft_float_transform_communicator(transform, comm) & bind(C, name="spfft_float_transform_communicator_fortran") use iso_c_binding type(c_ptr), value :: transform integer(c_int), intent(out) :: comm end function integer(c_int) function spfft_float_multi_transform_forward(numTransforms, transforms,& inputLocations, outputPointers, scalingTypes) bind(C) use iso_c_binding integer(c_int), value :: numTransforms type(c_ptr), value :: transforms type(c_ptr), value :: inputLocations type(c_ptr), value :: outputPointers type(c_ptr), value :: scalingTypes end function integer(c_int) function spfft_float_multi_transform_backward(numTransforms, transforms,& inputPointers, outputLocations) bind(C) use iso_c_binding integer(c_int), value :: numTransforms type(c_ptr), value :: transforms type(c_ptr), value :: inputPointers type(c_ptr), value :: outputLocations end function end interface end SpFFT-0.9.10/include/spfft/spfft.h000066400000000000000000000033411357247005600165750ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SPFFT_H #define SPFFT_SPFFT_H #include "spfft/config.h" #include "spfft/grid.h" #include "spfft/grid_float.h" #include "spfft/transform.h" #include "spfft/transform_float.h" #endif SpFFT-0.9.10/include/spfft/spfft.hpp000066400000000000000000000034751357247005600171450ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SPFFT_HPP #define SPFFT_SPFFT_HPP #include "spfft/config.h" #include "spfft/grid.hpp" #include "spfft/grid_float.hpp" #include "spfft/multi_transform.hpp" #include "spfft/multi_transform_float.hpp" #include "spfft/transform.hpp" #include "spfft/transform_float.hpp" #endif SpFFT-0.9.10/include/spfft/transform.h000066400000000000000000000244151357247005600174730ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_H #define SPFFT_TRANSFORM_H #include "spfft/config.h" #include "spfft/errors.h" #include "spfft/grid.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif #ifdef __cplusplus extern "C" { #endif /** * Transform handle. */ typedef void* SpfftTransform; /** * Creates a transform from a grid handle. * * @param[out] transform Handle to the transform. * @param[in] grid Handle to the grid, with which the transform is created. * @param[in] processingUnit The processing unit type to use. Must be either SPFFT_PU_HOST or * SPFFT_PU_GPU and be supported by the grid itself. * @param[in] transformType The transform type (complex to complex or real to complex). Can be * SPFFT_TRANS_C2C or SPFFT_TRANS_R2C. * @param[in] dimX The dimension in x. The maximum allowed depends on the grid parameters. * @param[in] dimY The dimension in y. The maximum allowed depends on the grid parameters. * @param[in] dimZ The dimension in z. The maximum allowed depends on the grid parameters. * @param[in] localZLength The length in z in space domain of the local MPI rank. * @param[in] numLocalElements The number of elements in frequency domain of the local MPI * rank. * @param[in] indexFormat The index format. Only SPFFT_INDEX_TRIPLETS currently supported. * @param[in] indices Pointer to the frequency indices. Posive and negative indexing is supported. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_create(SpfftTransform* transform, SpfftGrid grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices); /** * Destroy a transform. * * @param[in] transform Handle to the transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_destroy(SpfftTransform transform); /** * Clone a transform. * * @param[in] transform Handle to the transform. * @param[out] newTransform Independent transform with the same parameters, but with new underlying * grid. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_clone(SpfftTransform transform, SpfftTransform* newTransform); /** * Execute a forward transform from space domain to frequency domain. * * @param[in] transform Handle to the transform. * @param[in] inputLocation The processing unit, to take the input from. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] output Pointer to memory, where the frequency domain elements are written to. Can * be located at Host or GPU memory (if GPU is set as processing unit). * @param[in] scaling Controls scaling of output. SPFFT_NO_SCALING to disable or * SPFFT_FULL_SCALING to scale by factor 1 / (dim_x() * dim_y() * dim_z()). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_forward(SpfftTransform transform, SpfftProcessingUnitType inputLocation, double* output, SpfftScalingType scaling); /** * Execute a backward transform from frequency domain to space domain. * * @param[in] transform Handle to the transform. * @param[in] input Input data in frequency domain. Must match the indices provided at transform * creation. Can be located at Host or GPU memory, if GPU is set as processing unit. * @param[in] outputLocation The processing unit, to place the output at. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_backward(SpfftTransform transform, const double* input, SpfftProcessingUnitType outputLocation); /** * Provides access to the space domain data. * * @param[in] transform Handle to the transform. * @param[in] dataLocation The processing unit to query for the data. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] data Pointer to space domain data on given processing unit. Alignment is guaranteed * to fulfill requirements for std::complex and C language complex types. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_get_space_domain(SpfftTransform transform, SpfftProcessingUnitType dataLocation, double** data); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimX Dimension in x. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_dim_x(SpfftTransform transform, int* dimX); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimY Dimension in y. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_dim_y(SpfftTransform transform, int* dimY); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimZ Dimension in z. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_dim_z(SpfftTransform transform, int* dimZ); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] localZLength size in z of the slice in space domain on the local MPI rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_local_z_length(SpfftTransform transform, int* localZLength); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] size Number of elements in the space domain slice held by the local MPI rank. */ SPFFT_EXPORT SpfftError spfft_transform_local_slice_size(SpfftTransform transform, int* size); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] offset Offset in z of the space domain slice held by the local MPI rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_local_z_offset(SpfftTransform transform, int* offset); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] globalSize Global number of elements in space domain. Equals dim_x() * dim_y() * * dim_z(). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_global_size(SpfftTransform transform, long long int* globalSize); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numLocalElements Number of local elements in frequency domain. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_num_local_elements(SpfftTransform transform, int* numLocalElements); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numGlobalElements Global number of elements in space domain. Equals dim_x() * dim_y() * * dim_z(). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_num_global_elements(SpfftTransform transform, long long int* numGlobalElements); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] deviceId The GPU device id used. Returns always 0, if no GPU support is enabled. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_device_id(SpfftTransform transform, int* deviceId); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numThreads The exact number of threads used by transforms created from this grid. May * be less than the maximum given to the constructor. Always 1, if not compiled with OpenMP support. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_num_threads(SpfftTransform transform, int* numThreads); #ifdef SPFFT_MPI /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] comm The internal MPI communicator. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_transform_communicator(SpfftTransform transform, MPI_Comm* comm); #endif #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/transform.hpp000066400000000000000000000166211357247005600200330ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_HPP #define SPFFT_TRANSFORM_HPP #include #include "spfft/config.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif namespace spfft { template class SPFFT_NO_EXPORT TransformInternal; class SPFFT_EXPORT Grid; template class SPFFT_NO_EXPORT MultiTransformInternal; template class SPFFT_NO_EXPORT GridInternal; /** * A transform in double precision with fixed dimensions. Shares memory with other transform created * from the same Grid object. */ class SPFFT_EXPORT Transform { public: using ValueType = double; /** * Default copy constructor. */ Transform(const Transform&) = default; /** * Default move constructor. */ Transform(Transform&&) = default; /** * Default copy operator. */ Transform& operator=(const Transform&) = default; /** * Default move operator. */ Transform& operator=(Transform&&) = default; /** * Clone transform. * * @return Independent transform with the same parameters, but with new underlying grid. */ Transform clone() const; /** * Access a transform parameter. * @return Type of transform. */ SpfftTransformType type() const; /** * Access a transform parameter. * @return Dimension in x. */ int dim_x() const; /** * Access a transform parameter. * @return Dimension in y. */ int dim_y() const; /** * Access a transform parameter. * @return Dimension in z. */ int dim_z() const; /** * Access a transform parameter. * @return Length in z of the space domain slice held by the local MPI rank. */ int local_z_length() const; /** * Access a transform parameter. * @return Offset in z of the space domain slice held by the local MPI rank. */ int local_z_offset() const; /** * Access a transform parameter. * @return Number of elements in the space domain slice held by the local MPI rank. */ int local_slice_size() const; /** * Access a transform parameter. * @return Global number of elements in space domain. Equals dim_x() * dim_y() * dim_z(). */ long long int global_size() const; /** * Access a transform parameter. * @return Number of elements in frequency domain. */ int num_local_elements() const; /** * Access a transform parameter. * @return Global number of elements in frequency domain. */ long long int num_global_elements() const; /** * Access a transform parameter. * @return The processing unit used for calculations. Can be SPFFT_PU_HOST or SPFFT_PU_GPU. */ SpfftProcessingUnitType processing_unit() const; /** * Access a transform parameter. * @return The GPU device id used. Returns always 0, if no GPU support is enabled. */ int device_id() const; /** * Access a transform parameter. * @return The exact number of threads used by transforms created from this grid. May be less than * the maximum given to the constructor. Always 1, if not compiled with OpenMP support. */ int num_threads() const; #ifdef SPFFT_MPI /** * Access a transform parameter. * @return The internal MPI communicator. */ MPI_Comm communicator() const; #endif /** * Provides access to the space domain data. * * @param[in] dataLocation The processing unit to query for the data. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @return Pointer to space domain data on given processing unit. Alignment is guaranteed to * fulfill requirements for std::complex and C language complex types. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ double* space_domain_data(SpfftProcessingUnitType dataLocation); /** * Execute a forward transform from space domain to frequency domain. * * @param[in] inputLocation The processing unit, to take the input from. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] output Pointer to memory, where the frequency domain elements are written to. Can * be located at Host or GPU memory (if GPU is set as processing unit). * @param[in] scaling Controls scaling of output. SPFFT_NO_SCALING to disable or * SPFFT_FULL_SCALING to scale by factor 1 / (dim_x() * dim_y() * dim_z()). * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ void forward(SpfftProcessingUnitType inputLocation, double* output, SpfftScalingType scaling = SPFFT_NO_SCALING); /** * Execute a backward transform from frequency domain to space domain. * * @param[in] input Input data in frequency domain. Must match the indices provided at transform * creation. Can be located at Host or GPU memory, if GPU is set as processing unit. * @param[in] outputLocation The processing unit, to place the output at. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ void backward(const double* input, SpfftProcessingUnitType outputLocation); private: /*! \cond PRIVATE */ friend Grid; friend MultiTransformInternal; SPFFT_NO_EXPORT Transform(const std::shared_ptr>& grid, SpfftProcessingUnitType executionUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType dataFormat, const int* indices); SPFFT_NO_EXPORT explicit Transform(std::shared_ptr> transform); std::shared_ptr> transform_; /*! \endcond */ }; } // namespace spfft #endif SpFFT-0.9.10/include/spfft/transform_float.h000066400000000000000000000254101357247005600206540ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_FLOAT_H #define SPFFT_TRANSFORM_FLOAT_H #include "spfft/config.h" #include "spfft/errors.h" #include "spfft/grid_float.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif #ifdef __cplusplus extern "C" { #endif /** * Transform handle. */ typedef void* SpfftFloatTransform; /** * Creates a single precision transform from a single precision grid handle. * * @param[out] transform Handle to the transform. * @param[in] grid Handle to the grid, with which the transform is created. * @param[in] processingUnit The processing unit type to use. Must be either SPFFT_PU_HOST or * SPFFT_PU_GPU and be supported by the grid itself. * @param[in] transformType The transform type (complex to complex or real to complex). Can be * SPFFT_TRANS_C2C or SPFFT_TRANS_R2C. * @param[in] dimX The dimension in x. The maximum allowed depends on the grid parameters. * @param[in] dimY The dimension in y. The maximum allowed depends on the grid parameters. * @param[in] dimZ The dimension in z. The maximum allowed depends on the grid parameters. * @param[in] localZLength The length in z in space domain of the local MPI rank. * @param[in] numLocalElements The number of elements in frequency domain of the local MPI * rank. * @param[in] indexFormat The index format. Only SPFFT_INDEX_TRIPLETS currently supported. * @param[in] indices Pointer to the frequency indices. Posive and negative indexing is supported. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_create( SpfftFloatTransform* transform, SpfftFloatGrid grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices); /** * Destroy a transform. * * @param[in] transform Handle to the transform. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_destroy(SpfftFloatTransform transform); /** * Clone a transform. * * @param[in] transform Handle to the transform. * @param[out] newTransform Independent transform with the same parameters, but with new underlying * grid. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_clone(SpfftFloatTransform transform, SpfftFloatTransform* newTransform); /** * Execute a forward transform from space domain to frequency domain. * * @param[in] transform Handle to the transform. * @param[in] inputLocation The processing unit, to take the input from. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] output Pointer to memory, where the frequency domain elements are written to. Can * be located at Host or GPU memory (if GPU is set as processing unit). * @param[in] scaling Controls scaling of output. SPFFT_NO_SCALING to disable or * SPFFT_FULL_SCALING to scale by factor 1 / (dim_x() * dim_y() * dim_z()). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_forward(SpfftFloatTransform transform, SpfftProcessingUnitType inputLocation, float* output, SpfftScalingType scaling); /** * Execute a backward transform from frequency domain to space domain. * * @param[in] transform Handle to the transform. * @param[in] input Input data in frequency domain. Must match the indices provided at transform * creation. Can be located at Host or GPU memory, if GPU is set as processing unit. * @param[in] outputLocation The processing unit, to place the output at. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_backward(SpfftFloatTransform transform, const float* input, SpfftProcessingUnitType outputLocation); /** * Provides access to the space domain data. * * @param[in] transform Handle to the transform. * @param[in] dataLocation The processing unit to query for the data. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] data Pointer to space domain data on given processing unit. Alignment is guaranteed * to fulfill requirements for std::complex and C language complex types. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_get_space_domain(SpfftFloatTransform transform, SpfftProcessingUnitType dataLocation, float** data); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimX Dimension in x. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_dim_x(SpfftFloatTransform transform, int* dimX); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimY Dimension in y. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_dim_y(SpfftFloatTransform transform, int* dimY); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] dimZ Dimension in z. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_dim_z(SpfftFloatTransform transform, int* dimZ); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] localZLength size in z of the slice in space domain on the local MPI rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_local_z_length(SpfftFloatTransform transform, int* localZLength); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] size Number of elements in the space domain slice held by the local MPI rank. */ SPFFT_EXPORT SpfftError spfft_float_transform_local_slice_size(SpfftFloatTransform transform, int* size); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] globalSize Global number of elements in space domain. Equals dim_x() * dim_y() * * dim_z(). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_global_size(SpfftFloatTransform transform, long long int* globalSize); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] offset Offset in z of the space domain slice held by the local MPI rank. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_local_z_offset(SpfftFloatTransform transform, int* offset); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numLocalElements Number of local elements in frequency domain. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_num_local_elements(SpfftFloatTransform transform, int* numLocalElements); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numGlobalElements Global number of elements in space domain. Equals dim_x() * dim_y() * * dim_z(). * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_num_global_elements(SpfftFloatTransform transform, long long int* numGlobalElements); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] deviceId The GPU device id used. Returns always 0, if no GPU support is enabled. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_device_id(SpfftFloatTransform transform, int* deviceId); /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] numThreads The exact number of threads used by transforms created from this grid. May * be less than the maximum given to the constructor. Always 1, if not compiled with OpenMP support. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_num_threads(SpfftFloatTransform transform, int* numThreads); #ifdef SPFFT_MPI /** * Access a transform parameter. * @param[in] transform Handle to the transform. * @param[out] comm The internal MPI communicator. * @return Error code or SPFFT_SUCCESS. */ SPFFT_EXPORT SpfftError spfft_float_transform_communicator(SpfftFloatTransform transform, MPI_Comm* comm); #endif #ifdef __cplusplus } #endif #endif SpFFT-0.9.10/include/spfft/transform_float.hpp000066400000000000000000000170661357247005600212240ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_FLOAT_HPP #define SPFFT_TRANSFORM_FLOAT_HPP #include #include "spfft/config.h" #include "spfft/types.h" #ifdef SPFFT_MPI #include #endif namespace spfft { template class SPFFT_NO_EXPORT TransformInternal; template class SPFFT_NO_EXPORT MultiTransformInternal; template class SPFFT_NO_EXPORT GridInternal; #ifdef SPFFT_SINGLE_PRECISION class SPFFT_EXPORT GridFloat; /** * A transform in single precision with fixed dimensions. Shares memory with other transform created * from the same Grid object. */ class SPFFT_EXPORT TransformFloat { public: using ValueType = float; /** * Default copy constructor. */ TransformFloat(const TransformFloat&) = default; /** * Default move constructor. */ TransformFloat(TransformFloat&&) = default; /** * Default copy operator. */ TransformFloat& operator=(const TransformFloat&) = default; /** * Default move operator. */ TransformFloat& operator=(TransformFloat&&) = default; /** * Clone transform. * * @return Independent transform with the same parameters, but with new underlying grid. */ TransformFloat clone() const; /** * Access a transform parameter. * @return Type of transform. */ SpfftTransformType type() const; /** * Access a transform parameter. * @return Dimension in x. */ int dim_x() const; /** * Access a transform parameter. * @return Dimension in y. */ int dim_y() const; /** * Access a transform parameter. * @return Dimension in z. */ int dim_z() const; /** * Access a transform parameter. * @return Length in z of the space domain slice held by the local MPI rank. */ int local_z_length() const; /** * Access a transform parameter. * @return Offset in z of the space domain slice held by the local MPI rank. */ int local_z_offset() const; /** * Access a transform parameter. * @return Number of elements in the space domain slice held by the local MPI rank. */ int local_slice_size() const; /** * Access a transform parameter. * @return Global number of elements in space domain. Equals dim_x() * dim_y() * dim_z(). */ long long int global_size() const; /** * Access a transform parameter. * @return Number of elements in frequency domain. */ int num_local_elements() const; /** * Access a transform parameter. * @return Global number of elements in frequency domain. */ long long int num_global_elements() const; /** * Access a transform parameter. * @return The processing unit used for calculations. Can be SPFFT_PU_HOST or SPFFT_PU_GPU. */ SpfftProcessingUnitType processing_unit() const; /** * Access a transform parameter. * @return The GPU device id used. Returns always 0, if no GPU support is enabled. */ int device_id() const; /** * Access a transform parameter. * @return The exact number of threads used by transforms created from this grid. May be less than * the maximum given to the constructor. Always 1, if not compiled with OpenMP support. */ int num_threads() const; #ifdef SPFFT_MPI /** * Access a transform parameter. * @return The internal MPI communicator. */ MPI_Comm communicator() const; #endif /** * Provides access to the space domain data. * * @param[in] dataLocation The processing unit to query for the data. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @return Pointer to space domain data on given processing unit. Alignment is guaranteed to * fulfill requirements for std::complex and C language complex types. * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ float* space_domain_data(SpfftProcessingUnitType dataLocation); /** * Execute a forward transform from space domain to frequency domain. * * @param[in] inputLocation The processing unit, to take the input from. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @param[out] output Pointer to memory, where the frequency domain elements are written to. Can * be located at Host or GPU memory (if GPU is set as processing unit). * @param[in] scaling Controls scaling of output. SPFFT_NO_SCALING to disable or * SPFFT_FULL_SCALING to scale by factor 1 / (dim_x() * dim_y() * dim_z()). * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ void forward(SpfftProcessingUnitType inputLocation, float* output, SpfftScalingType scaling = SPFFT_NO_SCALING); /** * Execute a backward transform from frequency domain to space domain. * * @param[in] input Input data in frequency domain. Must match the indices provided at transform * creation. Can be located at Host or GPU memory, if GPU is set as processing unit. * @param[in] outputLocation The processing unit, to place the output at. Can be SPFFT_PU_HOST or * SPFFT_PU_GPU (if GPU is set as execution unit). * @throw GenericError SpFFT error. Can be a derived type. * @throw std::exception Error from standard library calls. Can be a derived type. */ void backward(const float* input, SpfftProcessingUnitType outputLocation); private: /*! \cond PRIVATE */ friend GridFloat; friend MultiTransformInternal; SPFFT_NO_EXPORT TransformFloat(const std::shared_ptr>& grid, SpfftProcessingUnitType executionUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType dataFormat, const int* indices); SPFFT_NO_EXPORT explicit TransformFloat(std::shared_ptr> transform); std::shared_ptr> transform_; /*! \endcond */ }; #endif } // namespace spfft #endif SpFFT-0.9.10/include/spfft/types.h000066400000000000000000000063721357247005600166260ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TYPES_H #define SPFFT_TYPES_H #include "spfft/config.h" enum SpfftExchangeType { /** * Default exchange. Equivalent to SPFFT_EXCH_COMPACT_BUFFERED. */ SPFFT_EXCH_DEFAULT, /** * Exchange based on MPI_Alltoall. */ SPFFT_EXCH_BUFFERED, /** * Exchange based on MPI_Alltoall in single precision. * Slight accuracy loss for double precision transforms due to conversion to float prior to MPI * exchange. */ SPFFT_EXCH_BUFFERED_FLOAT, /** * Exchange based on MPI_Alltoallv. */ SPFFT_EXCH_COMPACT_BUFFERED, /** * Exchange based on MPI_Alltoallv in single precision. * Slight accuracy loss for double precision transforms due to conversion to float prior to MPI * exchange. */ SPFFT_EXCH_COMPACT_BUFFERED_FLOAT, /** * Exchange based on MPI_Alltoallw. */ SPFFT_EXCH_UNBUFFERED }; /** * Processing unit type */ enum SpfftProcessingUnitType { /** * HOST */ SPFFT_PU_HOST = 1, /** * GPU */ SPFFT_PU_GPU = 2 }; enum SpfftIndexFormatType { /** * Triplets of x,y,z frequency indices */ SPFFT_INDEX_TRIPLETS }; enum SpfftTransformType { /** * Complex-to-Complex transform */ SPFFT_TRANS_C2C, /** * Real-to-Complex transform */ SPFFT_TRANS_R2C }; enum SpfftScalingType { /** * No scaling */ SPFFT_NO_SCALING, /** * Full scaling */ SPFFT_FULL_SCALING }; #ifndef __cplusplus /*! \cond PRIVATE */ // C only typedef enum SpfftExchangeType SpfftExchangeType; typedef enum SpfftProcessingUnitType SpfftProcessingUnitType; typedef enum SpfftTransformType SpfftTransformType; typedef enum SpfftIndexFormatType SpfftIndexFormatType; typedef enum SpfftScalingType SpfftScalingType; /*! \endcond */ #endif // cpp #endif SpFFT-0.9.10/src/000077500000000000000000000000001357247005600133235ustar00rootroot00000000000000SpFFT-0.9.10/src/CMakeLists.txt000066400000000000000000000140041357247005600160620ustar00rootroot00000000000000set(SPFFT_SOURCE_FILES memory/aligned_allocation.cpp timing/timing.cpp timing/rt_graph.cpp parameters/parameters.cpp execution/execution_host.cpp spfft/transform.cpp spfft/transform_internal.cpp spfft/multi_transform.cpp spfft/grid.cpp spfft/grid_internal.cpp ) if(SPFFT_SINGLE_PRECISION) list(APPEND SPFFT_SOURCE_FILES spfft/transform_float.cpp spfft/multi_transform_float.cpp spfft/grid_float.cpp ) endif() set(SPFFT_GPU_KERNELS) if(SPFFT_CUDA OR SPFFT_ROCM) list(APPEND SPFFT_GPU_KERNELS transpose/gpu_kernels/local_transpose_kernels.cu compression/gpu_kernels/compression_kernels.cu symmetry/gpu_kernels/symmetry_kernels.cu transpose/gpu_kernels/buffered_kernels.cu transpose/gpu_kernels/compact_buffered_kernels.cu ) list(APPEND SPFFT_SOURCE_FILES execution/execution_gpu.cpp gpu_util/gpu_fft_api.cpp ) if(SPFFT_MPI) list(APPEND SPFFT_SOURCE_FILES transpose/transpose_mpi_buffered_gpu.cpp transpose/transpose_mpi_compact_buffered_gpu.cpp transpose/transpose_mpi_unbuffered_gpu.cpp ) endif() endif() if(SPFFT_CUDA) list(APPEND SPFFT_SOURCE_FILES ${SPFFT_GPU_KERNELS}) endif() if(SPFFT_ROCM) rocm_generate_objects(ROCM_OBJECTS ROCM_TARGETS ${SPFFT_GPU_KERNELS} INCLUDE_DIRS ${SPFFT_INCLUDE_DIRS} ${SPFFT_EXTERNAL_INCLUDE_DIRS} FLAGS -fPIC -fno-gpu-rdc --amdgpu-target=gfx803 --amdgpu-target=gfx900 --amdgpu-target=gfx906) list(APPEND SPFFT_SOURCE_FILES ${ROCM_OBJECTS}) endif() if(SPFFT_MPI) list(APPEND SPFFT_SOURCE_FILES transpose/transpose_mpi_buffered_host.cpp transpose/transpose_mpi_compact_buffered_host.cpp transpose/transpose_mpi_unbuffered_host.cpp ) endif() add_library(spfft ${SPFFT_LIBRARY_TYPE} ${SPFFT_SOURCE_FILES} ) set_property(TARGET spfft PROPERTY VERSION ${SPFFT_VERSION}) set_property(TARGET spfft PROPERTY SOVERSION ${SPFFT_SO_VERSION}) # build fortran module if(SPFFT_FORTRAN) add_library(spfft_fortran OBJECT ${PROJECT_SOURCE_DIR}/include/spfft/spfft.f90) endif() # Don't export any symbols of external static libaries. Only works on linux. if(UNIX AND NOT APPLE) if(${CMAKE_VERSION} VERSION_LESS "3.13.5") set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} "-Wl,--exclude-libs,ALL") else() target_link_options(spfft PRIVATE "-Wl,--exclude-libs,ALL") endif() endif() # add depency on ojbect generating targets for ROCM if(SPFFT_ROCM) add_dependencies(spfft ${ROCM_TARGETS}) endif() # generate export header to control symbol visibility include(GenerateExportHeader) generate_export_header(spfft) configure_file("${CMAKE_CURRENT_BINARY_DIR}/spfft_export.h" "${PROJECT_BINARY_DIR}/spfft/spfft_export.h" COPYONLY ) target_compile_options(spfft PRIVATE ${SPFFT_DEFINITIONS} ${SPFFT_EXTERNAL_COMPILE_OPTIONS}) target_include_directories(spfft PRIVATE ${SPFFT_EXTERNAL_INCLUDE_DIRS}) target_include_directories(spfft PRIVATE ${PROJECT_SOURCE_DIR}/include) target_include_directories(spfft PRIVATE ${PROJECT_SOURCE_DIR}/src) target_include_directories(spfft PRIVATE ${PROJECT_BINARY_DIR}) target_link_libraries(spfft PRIVATE ${SPFFT_EXTERNAL_LIBS}) target_include_directories(spfft INTERFACE ${SPFFT_INTERFACE_INCLUDE_DIRS}) target_include_directories(spfft INTERFACE $) # for install(EXPORT ...) target_include_directories(spfft INTERFACE $ $) # for export(...) # set list of interface libries for exporting (workaround for bug with exporting static library: # cmake uses MPI_CXX and OpenMP::OpenMP_CXX targets, which are not known externally) if(POLICY CMP0022) set_target_properties(spfft PROPERTIES INTERFACE_LINK_LIBRARIES "${SPFFT_INTERFACE_LIBS}") else() set_target_properties(spfft PROPERTIES LINK_INTERFACE_LIBRARIES "${SPFFT_INTERFACE_LIBS}") endif() # set packge config names if(SPFFT_STATIC) set(SPFFT_VERSION_FILE "SpFFTStaticConfigVersion.cmake") set(SPFFT_CONFIG_FILE "SpFFTStaticConfig.cmake") set(SPFFT_TARGETS_FILE "SpFFTStaticTargets.cmake") else() set(SPFFT_VERSION_FILE "SpFFTSharedConfigVersion.cmake") set(SPFFT_CONFIG_FILE "SpFFTSharedConfig.cmake") set(SPFFT_TARGETS_FILE "SpFFTSharedTargets.cmake") endif() # generate cmake package include(CMakePackageConfigHelpers) write_basic_package_version_file( "${PROJECT_BINARY_DIR}/${SPFFT_VERSION_FILE}" VERSION ${Upstream_VERSION} COMPATIBILITY AnyNewerVersion ) export(TARGETS spfft NAMESPACE SpFFT:: FILE ${PROJECT_BINARY_DIR}/${SPFFT_TARGETS_FILE}) configure_file(${PROJECT_SOURCE_DIR}/cmake/${SPFFT_CONFIG_FILE} "${PROJECT_BINARY_DIR}/${SPFFT_CONFIG_FILE}" @ONLY ) configure_file(${PROJECT_SOURCE_DIR}/cmake/SpFFTConfig.cmake "${PROJECT_BINARY_DIR}/SpFFTConfig.cmake" COPYONLY ) configure_file(${PROJECT_SOURCE_DIR}/cmake/SpFFTConfigVersion.cmake "${PROJECT_BINARY_DIR}/SpFFTConfigVersion.cmake" COPYONLY ) configure_file(${PROJECT_SOURCE_DIR}/cmake/SpFFTTargets.cmake "${PROJECT_BINARY_DIR}/SpFFTTargets.cmake" COPYONLY ) configure_file(${PROJECT_SOURCE_DIR}/cmake/SpFFT.pc.in "${PROJECT_BINARY_DIR}/SpFFT.pc" @ONLY ) # installation commands if(SPFFT_INSTALL) install(TARGETS spfft DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT SpFFTTargets) install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/spfft DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp" PATTERN "*.f90") install(FILES ${PROJECT_BINARY_DIR}/spfft/config.h "${PROJECT_BINARY_DIR}/spfft/spfft_export.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/spfft) install(EXPORT SpFFTTargets NAMESPACE SpFFT:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SpFFT FILE ${SPFFT_TARGETS_FILE}) install( FILES "${PROJECT_BINARY_DIR}/SpFFTConfig.cmake" "${PROJECT_BINARY_DIR}/SpFFTTargets.cmake" "${PROJECT_BINARY_DIR}/SpFFTConfigVersion.cmake" "${PROJECT_BINARY_DIR}/${SPFFT_CONFIG_FILE}" "${PROJECT_BINARY_DIR}/${SPFFT_VERSION_FILE}" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SpFFT ) install(FILES ${PROJECT_BINARY_DIR}/SpFFT.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) if(SPFFT_FORTRAN) install(FILES ${PROJECT_BINARY_DIR}/src/spfft.mod DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/spfft) endif() endif() SpFFT-0.9.10/src/compression/000077500000000000000000000000001357247005600156645ustar00rootroot00000000000000SpFFT-0.9.10/src/compression/compression_gpu.hpp000066400000000000000000000071751357247005600216230ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_COMPRESSION_GPU_HPP #define SPFFT_COMPRESSION_GPU_HPP #include #include #include #include #include "compression/gpu_kernels/compression_kernels.hpp" #include "compression/indices.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "gpu_util/gpu_transfer.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "util/common_types.hpp" #include "util/type_check.hpp" namespace spfft { // Handles packing and unpacking of sparse frequency values for single or double precision on GPU class CompressionGPU { public: CompressionGPU(const std::shared_ptr& param) : indicesGPU_( param->local_value_indices().size()) { // stream MUST synchronize with default stream copy_to_gpu(param->local_value_indices(), indicesGPU_); } // Pack values into output buffer template auto compress(const GPUStreamHandle& stream, const GPUArrayView2D::type> input, T* output, const bool useScaling, const T scalingFactor = 1.0) -> void { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); compress_gpu(stream.get(), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), input, output, useScaling, scalingFactor); } // Unpack values into z-stick collection template auto decompress(const GPUStreamHandle& stream, const T* input, GPUArrayView2D::type> output) -> void { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); gpu::check_status(gpu::memset_async( static_cast(output.data()), 0, output.size() * sizeof(typename decltype(output)::ValueType), stream.get())); decompress_gpu(stream.get(), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), input, output); } private: GPUArray indicesGPU_; }; } // namespace spfft #endif SpFFT-0.9.10/src/compression/compression_host.hpp000066400000000000000000000077371357247005600220110ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_COMPRESSION_HOST_HPP #define SPFFT_COMPRESSION_HOST_HPP #include #include #include #include #include "compression/indices.hpp" #include "memory/host_array_const_view.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" namespace spfft { // Handles packing and unpacking of sparse frequency values for single or double precision on Host class CompressionHost { public: explicit CompressionHost(const std::shared_ptr& param) : param_(param) {} // Pack values into output buffer template auto compress(const HostArrayView2D> input2d, T* output, bool useScaling, const T scalingFactor = 1.0) const -> void { const auto& indices = param_->local_value_indices(); auto input = HostArrayConstView1D>(input2d.data(), input2d.size(), input2d.pinned()); if (useScaling) { SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < indices.size(); ++i) { const auto value = scalingFactor * input(indices[i]); output[2 * i] = value.real(); output[2 * i + 1] = value.imag(); } } else { SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < indices.size(); ++i) { const auto value = input(indices[i]); output[2 * i] = value.real(); output[2 * i + 1] = value.imag(); } } } // Unpack values into z-stick collection template auto decompress(const T* input, HostArrayView2D> output2d) const -> void { const auto& indices = param_->local_value_indices(); auto output = HostArrayView1D>(output2d.data(), output2d.size(), output2d.pinned()); // ensure values are padded with zeros SPFFT_OMP_PRAGMA("omp for schedule(static)") // implicit barrier for (SizeType stick = 0; stick < output2d.dim_outer(); ++stick) { std::memset(static_cast(&output2d(stick, 0)), 0, sizeof(typename decltype(output2d)::ValueType) * output2d.dim_inner()); } SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < indices.size(); ++i) { output(indices[i]) = std::complex(input[2 * i], input[2 * i + 1]); } } private: std::shared_ptr param_; }; } // namespace spfft #endif SpFFT-0.9.10/src/compression/gpu_kernels/000077500000000000000000000000001357247005600202025ustar00rootroot00000000000000SpFFT-0.9.10/src/compression/gpu_kernels/compression_kernels.cu000066400000000000000000000143061357247005600246230ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_kernel_parameter.hpp" #include "gpu_util/gpu_runtime.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array_const_view.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { template __global__ static void decompress_kernel( const GPUArrayConstView1D indices, const T* input, GPUArrayView1D::type> output) { // const int stride = gridDim.x * blockDim.x; for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < indices.size(); idx += gridDim.x * blockDim.x) { const int valueIdx = indices(idx); typename gpu::fft::ComplexType::type value; value.x = input[2 * idx]; value.y = input[2 * idx + 1]; output(valueIdx) = value; } } auto decompress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, const double* input, GPUArrayView2D::type> output) -> void { assert(indices.size() <= output.size()); const dim3 threadBlock(gpu::BlockSizeMedium); const dim3 threadGrid(std::min( static_cast((indices.size() + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); // const dim3 threadGrid(indices.size() < 4 ? 1 : indices.size() / 4); launch_kernel(decompress_kernel, threadGrid, threadBlock, 0, stream, indices, input, create_1d_view(output, 0, output.size())); } auto decompress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, const float* input, GPUArrayView2D::type> output) -> void { assert(indices.size() <= output.size()); const dim3 threadBlock(gpu::BlockSizeMedium); const dim3 threadGrid(std::min( static_cast((indices.size() + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); launch_kernel(decompress_kernel, threadGrid, threadBlock, 0, stream, indices, input, create_1d_view(output, 0, output.size())); } template __global__ static void compress_kernel( const GPUArrayConstView1D indices, GPUArrayConstView1D::type> input, T* output) { for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < indices.size(); idx += gridDim.x * blockDim.x) { const int valueIdx = indices(idx); const auto value = input(valueIdx); output[2 * idx] = value.x; output[2 * idx + 1] = value.y; } } template __global__ static void compress_kernel_scaled( const GPUArrayConstView1D indices, GPUArrayConstView1D::type> input, T* output, const T scalingFactor) { for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < indices.size(); idx += gridDim.x * blockDim.x) { const int valueIdx = indices(idx); const auto value = input(valueIdx); output[2 * idx] = scalingFactor * value.x; output[2 * idx + 1] = scalingFactor * value.y; } } auto compress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, GPUArrayView2D::type> input, double* output, const bool useScaling, const double scalingFactor) -> void { const dim3 threadBlock(gpu::BlockSizeMedium); const dim3 threadGrid(std::min( static_cast((indices.size() + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); if (useScaling) { launch_kernel(compress_kernel_scaled, threadGrid, threadBlock, 0, stream, indices, create_1d_view(input, 0, input.size()), output, scalingFactor); } else { launch_kernel(compress_kernel, threadGrid, threadBlock, 0, stream, indices, create_1d_view(input, 0, input.size()), output); } } auto compress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, GPUArrayView2D::type> input, float* output, const bool useScaling, const float scalingFactor) -> void { const dim3 threadBlock(gpu::BlockSizeMedium); const dim3 threadGrid(std::min( static_cast((indices.size() + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); if (useScaling) { launch_kernel(compress_kernel_scaled, threadGrid, threadBlock, 0, stream, indices, create_1d_view(input, 0, input.size()), output, scalingFactor); } else { launch_kernel(compress_kernel, threadGrid, threadBlock, 0, stream, indices, create_1d_view(input, 0, input.size()), output); } } } // namespace spfft SpFFT-0.9.10/src/compression/gpu_kernels/compression_kernels.hpp000066400000000000000000000053201357247005600247770ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef COMPRESSION_KERNELS_HPP #define COMPRESSION_KERNELS_HPP #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_runtime_api.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { auto decompress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, const double* input, GPUArrayView2D::type> output) -> void; auto decompress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, const float* input, GPUArrayView2D::type> output) -> void; auto compress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, GPUArrayView2D::type> input, double* output, const bool useScaling, const double scalingFactor) -> void; auto compress_gpu(const gpu::StreamType stream, const GPUArrayView1D& indices, GPUArrayView2D::type> input, float* output, const bool useScaling, const float scalingFactor) -> void; } // namespace spfft #endif SpFFT-0.9.10/src/compression/indices.hpp000066400000000000000000000155421357247005600200220ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_INDICES_HPP #define SPFFT_INDICES_HPP #include #include #include #include #include #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #endif namespace spfft { // convert [-N, N) frequency index to [0, N) for FFT input inline auto to_storage_index(const int dim, const int index) -> int { if (index < 0) { return dim + index; } else { return index; } } #ifdef SPFFT_MPI inline auto create_distributed_transform_indices(const MPICommunicatorHandle& comm, std::vector localSticks) -> std::vector> { std::vector sendRequests(comm.size()); constexpr int tag = 442; // random tag (must be less than 32768) // send local stick indices for (int r = 0; r < static_cast(comm.size()); ++r) { if (r != static_cast(comm.rank())) { mpi_check_status(MPI_Isend(localSticks.data(), localSticks.size(), MPI_INT, r, tag, comm.get(), &(sendRequests[r]))); } } std::vector> globalXYIndices(comm.size()); // recv all other stick indices for (int r = 0; r < static_cast(comm.size()); ++r) { if (r != static_cast(comm.rank())) { // get recv count MPI_Status status; MPI_Probe(r, tag, comm.get(), &status); int recvCount = 0; MPI_Get_count(&status, MPI_INT, &recvCount); // recv data globalXYIndices[r].resize(recvCount); MPI_Recv(globalXYIndices[r].data(), recvCount, MPI_INT, r, tag, comm.get(), MPI_STATUS_IGNORE); } } // wait for all sends to finish for (int r = 0; r < static_cast(comm.size()); ++r) { if (r != static_cast(comm.rank())) { MPI_Wait(&(sendRequests[r]), MPI_STATUS_IGNORE); } } // move local sticks into transform indices object AFTER sends are finished globalXYIndices[comm.rank()] = std::move(localSticks); return globalXYIndices; } #endif inline auto check_stick_duplicates(const std::vector>& indicesPerRank) -> void { // check for z-sticks indices std::set globalXYIndices; for (const auto& rankIndices : indicesPerRank) { for (const auto& index : rankIndices) { if (globalXYIndices.count(index)) { throw DuplicateIndicesError(); } globalXYIndices.insert(index); } } } // convert index triplets for every value into stick/z indices and z-stick index pairs. inline auto convert_index_triplets(const bool hermitianSymmetry, const int dimX, const int dimY, const int dimZ, const int numValues, const int* xIndices, const int* yIndices, const int* zIndices, const int stride) -> std::pair, std::vector> { if (static_cast(numValues) > static_cast(dimX) * static_cast(dimY) * static_cast(dimZ)) { throw InvalidParameterError(); } // check if indices are non-negative or centered bool centeredIndices = false; for (int i = 0; i < numValues; ++i) { if (xIndices[i * stride] < 0 || yIndices[i * stride] < 0 || zIndices[i * stride] < 0) { centeredIndices = true; break; } } const int maxX = (hermitianSymmetry || centeredIndices ? dimX / 2 + 1 : dimX) - 1; const int maxY = (centeredIndices ? dimY / 2 + 1 : dimY) - 1; const int maxZ = (centeredIndices ? dimZ / 2 + 1 : dimZ) - 1; const int minX = hermitianSymmetry ? 0 : maxX - dimX + 1; const int minY = maxY - dimY + 1; const int minZ = maxZ - dimZ + 1; // check if indices are inside bounds for (int i = 0; i < numValues; ++i) { if (xIndices[i * stride] < minX || xIndices[i * stride] > maxX) throw InvalidIndicesError(); if (yIndices[i * stride] < minY || yIndices[i * stride] > maxY) throw InvalidIndicesError(); if (zIndices[i * stride] < minZ || zIndices[i * stride] > maxZ) throw InvalidIndicesError(); } // store all unique xy index pairs in an ordered container std::map sortedXYIndices; // key = index in xy-plane, value = stick index for (int i = 0; i < numValues; ++i) { const auto x = to_storage_index(dimX, xIndices[i * stride]); const auto y = to_storage_index(dimY, yIndices[i * stride]); sortedXYIndices[x * dimY + y] = 0; } // assign z-stick indices int count = 0; for (auto& pair : sortedXYIndices) { pair.second = count; ++count; } // store index for each element. Each z-stick is continous std::vector valueIndices; valueIndices.reserve(numValues); for (int i = 0; i < numValues; ++i) { const auto x = to_storage_index(dimX, xIndices[i * stride]); const auto y = to_storage_index(dimY, yIndices[i * stride]); const auto z = to_storage_index(dimZ, zIndices[i * stride]); valueIndices.emplace_back(sortedXYIndices[x * dimY + y] * dimZ + z); } // store ordered unique xy-index pairs std::vector stickIndices; stickIndices.reserve(sortedXYIndices.size()); for (auto& pair : sortedXYIndices) { stickIndices.emplace_back(pair.first); } return {std::move(valueIndices), std::move(stickIndices)}; } } // namespace spfft #endif SpFFT-0.9.10/src/execution/000077500000000000000000000000001357247005600153265ustar00rootroot00000000000000SpFFT-0.9.10/src/execution/execution_gpu.cpp000066400000000000000000000421671357247005600207220ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "execution/execution_gpu.hpp" #include "fft/transform_1d_gpu.hpp" #include "fft/transform_2d_gpu.hpp" #include "fft/transform_real_2d_gpu.hpp" #include "gpu_util/gpu_pointer_translation.hpp" #include "gpu_util/gpu_transfer.hpp" #include "memory/array_view_utility.hpp" #include "parameters/parameters.hpp" #include "symmetry/symmetry_gpu.hpp" #include "timing/timing.hpp" #include "transpose/transpose_gpu.hpp" #include "transpose/transpose_mpi_buffered_gpu.hpp" #include "transpose/transpose_mpi_compact_buffered_gpu.hpp" #include "transpose/transpose_mpi_unbuffered_gpu.hpp" namespace spfft { template ExecutionGPU::ExecutionGPU(const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2, GPUArray::type>& gpuArray1, GPUArray::type>& gpuArray2, const std::shared_ptr>& fftWorkBuffer) : stream_(false), event_(false), numThreads_(numThreads), scalingFactor_(static_cast( 1.0 / static_cast(param->dim_x() * param->dim_y() * param->dim_z()))), zStickSymmetry_(new Symmetry()), planeSymmetry_(new Symmetry()) { const SizeType numLocalZSticks = param->num_z_sticks(0); // frequency data with z-sticks freqDomainDataGPU_ = create_2d_view(gpuArray1, 0, numLocalZSticks, param->dim_z()); freqDomainCompressedDataGPU_ = GPUArrayView1D(reinterpret_cast(gpuArray2.data()), param->local_value_indices().size() * 2, gpuArray2.device_id()); // Z if (numLocalZSticks > 0) { transformZ_ = std::unique_ptr( new Transform1DGPU(freqDomainDataGPU_, stream_, fftWorkBuffer)); if (param->transform_type() == SPFFT_TRANS_R2C) { zStickSymmetry_.reset(new StickSymmetryGPU( stream_, GPUArrayView1D::type>( freqDomainDataGPU_.data() + freqDomainDataGPU_.index(param->zero_zero_stick_index(), 0), freqDomainDataGPU_.dim_inner(), freqDomainDataGPU_.device_id()))); } } if (numLocalZSticks > 0 && param->local_value_indices().size() > 0) { compression_.reset(new CompressionGPU(param)); } // Transpose auto freqDomainXYGPU = create_3d_view(gpuArray2, 0, param->dim_z(), param->dim_y(), param->dim_x_freq()); // must not overlap with z-sticks transpose_.reset(new TransposeGPU(param, stream_, freqDomainXYGPU, freqDomainDataGPU_)); // XY if (param->num_xy_planes(0) > 0) { if (param->transform_type() == SPFFT_TRANS_R2C) { planeSymmetry_.reset(new PlaneSymmetryGPU(stream_, freqDomainXYGPU)); // NOTE: param->dim_x() != param->dim_x_freq() spaceDomainDataExternalHost_ = create_new_type_3d_view(array1, param->dim_z(), param->dim_y(), param->dim_x()); spaceDomainDataExternalGPU_ = create_new_type_3d_view(gpuArray1, param->dim_z(), param->dim_y(), param->dim_x()); transformXY_ = std::unique_ptr(new TransformReal2DGPU( spaceDomainDataExternalGPU_, freqDomainXYGPU, stream_, fftWorkBuffer)); } else { spaceDomainDataExternalHost_ = create_new_type_3d_view( array1, param->dim_z(), param->dim_y(), 2 * param->dim_x_freq()); spaceDomainDataExternalGPU_ = create_new_type_3d_view( freqDomainXYGPU, param->dim_z(), param->dim_y(), 2 * param->dim_x_freq()); transformXY_ = std::unique_ptr( new Transform2DGPU(freqDomainXYGPU, stream_, fftWorkBuffer)); } } } #ifdef SPFFT_MPI template ExecutionGPU::ExecutionGPU(MPICommunicatorHandle comm, const SpfftExchangeType exchangeType, const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2, GPUArray::type>& gpuArray1, GPUArray::type>& gpuArray2, const std::shared_ptr>& fftWorkBuffer) : stream_(false), event_(false), numThreads_(numThreads), scalingFactor_(static_cast( 1.0 / static_cast(param->dim_x() * param->dim_y() * param->dim_z()))), zStickSymmetry_(new Symmetry()), planeSymmetry_(new Symmetry()) { assert(array1.data() != array2.data()); assert(gpuArray1.data() != gpuArray2.data()); assert(gpuArray1.device_id() == gpuArray2.device_id()); const SizeType numLocalZSticks = param->num_z_sticks(comm.rank()); const SizeType numLocalXYPlanes = param->num_xy_planes(comm.rank()); freqDomainDataGPU_ = create_2d_view(gpuArray1, 0, numLocalZSticks, param->dim_z()); freqDomainCompressedDataGPU_ = GPUArrayView1D(reinterpret_cast(gpuArray2.data()), param->local_value_indices().size() * 2, gpuArray2.device_id()); auto freqDomainXYGPU = create_3d_view(gpuArray2, 0, numLocalXYPlanes, param->dim_y(), param->dim_x_freq()); // must not overlap with z-sticks // Z if (numLocalZSticks > 0) { transformZ_ = std::unique_ptr( new Transform1DGPU(freqDomainDataGPU_, stream_, fftWorkBuffer)); if (param->transform_type() == SPFFT_TRANS_R2C && param->zero_zero_stick_index() < freqDomainDataGPU_.dim_outer()) { zStickSymmetry_.reset(new StickSymmetryGPU( stream_, GPUArrayView1D::type>( freqDomainDataGPU_.data() + freqDomainDataGPU_.index(param->zero_zero_stick_index(), 0), freqDomainDataGPU_.dim_inner(), freqDomainDataGPU_.device_id()))); } } if (numLocalZSticks > 0) { compression_.reset(new CompressionGPU(param)); } // XY if (numLocalXYPlanes > 0) { if (param->transform_type() == SPFFT_TRANS_R2C) { // NOTE: param->dim_x() != param->dim_x_freq() spaceDomainDataExternalHost_ = create_new_type_3d_view(array1, numLocalXYPlanes, param->dim_y(), param->dim_x()); spaceDomainDataExternalGPU_ = create_new_type_3d_view(gpuArray1, numLocalXYPlanes, param->dim_y(), param->dim_x()); transformXY_ = std::unique_ptr(new TransformReal2DGPU( spaceDomainDataExternalGPU_, freqDomainXYGPU, stream_, fftWorkBuffer)); planeSymmetry_.reset(new PlaneSymmetryGPU(stream_, freqDomainXYGPU)); } else { spaceDomainDataExternalHost_ = create_new_type_3d_view( array1, numLocalXYPlanes, param->dim_y(), 2 * param->dim_x_freq()); spaceDomainDataExternalGPU_ = create_new_type_3d_view( freqDomainXYGPU, numLocalXYPlanes, param->dim_y(), 2 * param->dim_x_freq()); transformXY_ = std::unique_ptr( new Transform2DGPU(freqDomainXYGPU, stream_, fftWorkBuffer)); } } switch (exchangeType) { case SpfftExchangeType::SPFFT_EXCH_UNBUFFERED: { auto freqDomainDataHost = create_2d_view(array1, 0, numLocalZSticks, param->dim_z()); auto freqDomainXYHost = create_3d_view(array2, 0, numLocalXYPlanes, param->dim_y(), param->dim_x_freq()); transpose_.reset( new TransposeMPIUnbufferedGPU(param, comm, freqDomainXYHost, freqDomainXYGPU, stream_, freqDomainDataHost, freqDomainDataGPU_, stream_)); } break; case SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED: { const auto bufferZSize = param->total_num_xy_planes() * param->num_z_sticks(comm.rank()); const auto bufferXYSize = param->total_num_z_sticks() * param->num_xy_planes(comm.rank()); auto transposeBufferZ = create_1d_view(array2, 0, bufferZSize); auto transposeBufferZGPU = create_1d_view(gpuArray2, 0, bufferZSize); auto transposeBufferXY = create_1d_view(array1, 0, bufferXYSize); auto transposeBufferXYGPU = create_1d_view(gpuArray1, 0, bufferXYSize); transpose_.reset(new TransposeMPICompactBufferedGPU( param, comm, transposeBufferXY, freqDomainXYGPU, transposeBufferXYGPU, stream_, transposeBufferZ, freqDomainDataGPU_, transposeBufferZGPU, stream_)); } break; case SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED_FLOAT: { const auto bufferZSize = param->total_num_xy_planes() * param->num_z_sticks(comm.rank()); const auto bufferXYSize = param->total_num_z_sticks() * param->num_xy_planes(comm.rank()); auto transposeBufferZ = create_1d_view(array2, 0, bufferZSize); auto transposeBufferZGPU = create_1d_view(gpuArray2, 0, bufferZSize); auto transposeBufferXY = create_1d_view(array1, 0, bufferXYSize); auto transposeBufferXYGPU = create_1d_view(gpuArray1, 0, bufferXYSize); transpose_.reset(new TransposeMPICompactBufferedGPU( param, comm, transposeBufferXY, freqDomainXYGPU, transposeBufferXYGPU, stream_, transposeBufferZ, freqDomainDataGPU_, transposeBufferZGPU, stream_)); } break; case SpfftExchangeType::SPFFT_EXCH_BUFFERED: { const auto bufferSize = param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size(); auto transposeBufferZ = create_1d_view(array2, 0, bufferSize); auto transposeBufferZGPU = create_1d_view(gpuArray2, 0, bufferSize); auto transposeBufferXY = create_1d_view(array1, 0, bufferSize); auto transposeBufferXYGPU = create_1d_view(gpuArray1, 0, bufferSize); transpose_.reset(new TransposeMPIBufferedGPU( param, comm, transposeBufferXY, freqDomainXYGPU, transposeBufferXYGPU, stream_, transposeBufferZ, freqDomainDataGPU_, transposeBufferZGPU, stream_)); } break; case SpfftExchangeType::SPFFT_EXCH_BUFFERED_FLOAT: { const auto bufferSize = param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size(); auto transposeBufferZ = create_1d_view(array2, 0, bufferSize); auto transposeBufferZGPU = create_1d_view(gpuArray2, 0, bufferSize); auto transposeBufferXY = create_1d_view(array1, 0, bufferSize); auto transposeBufferXYGPU = create_1d_view(gpuArray1, 0, bufferSize); transpose_.reset(new TransposeMPIBufferedGPU( param, comm, transposeBufferXY, freqDomainXYGPU, transposeBufferXYGPU, stream_, transposeBufferZ, freqDomainDataGPU_, transposeBufferZGPU, stream_)); } break; default: throw InvalidParameterError(); } } // instatiate templates for float and double #endif template auto ExecutionGPU::forward_xy(const SpfftProcessingUnitType inputLocation) -> void { // Check for any preceding errors before starting execution if (gpu::get_last_error() != gpu::status::Success) { throw GPUPrecedingError(); } // XY if (transformXY_) { // Add explicit default stream synchronization event_.record(nullptr); event_.stream_wait(stream_.get()); if (inputLocation == SpfftProcessingUnitType::SPFFT_PU_HOST) { copy_to_gpu_async(stream_, spaceDomainDataExternalHost_, spaceDomainDataExternalGPU_); } transformXY_->forward(); } // transpose if (transformXY_) transpose_->pack_forward(); } template auto ExecutionGPU::forward_exchange(const bool nonBlockingExchange) -> void { HOST_TIMING_SCOPED("exchange_start") transpose_->exchange_forward_start(nonBlockingExchange); } template auto ExecutionGPU::forward_z(T* output, const SpfftScalingType scalingType) -> void { HOST_TIMING_START("exechange_fininalize"); transpose_->exchange_forward_finalize(); HOST_TIMING_STOP("exechange_fininalize"); if (transformZ_) transpose_->unpack_forward(); // Z if (transformZ_) transformZ_->forward(); // Compress if (compression_) { T* outputPtrHost = nullptr; T* outputPtrGPU = nullptr; std::tie(outputPtrHost, outputPtrGPU) = translate_gpu_pointer(output); if (outputPtrGPU == nullptr) { // output on HOST compression_->compress(stream_, freqDomainDataGPU_, freqDomainCompressedDataGPU_.data(), scalingType == SpfftScalingType::SPFFT_FULL_SCALING, scalingFactor_); gpu::check_status( gpu::memcpy_async(static_cast(outputPtrHost), static_cast(freqDomainCompressedDataGPU_.data()), freqDomainCompressedDataGPU_.size() * sizeof(decltype(*(freqDomainCompressedDataGPU_.data()))), gpu::flag::MemcpyDeviceToHost, stream_.get())); } else { // output on GPU compression_->compress(stream_, freqDomainDataGPU_, outputPtrGPU, scalingType == SpfftScalingType::SPFFT_FULL_SCALING, scalingFactor_); } } } template auto ExecutionGPU::backward_z(const T* input) -> void { // Check for any preceding errors before starting execution if (gpu::get_last_error() != gpu::status::Success) { throw GPUPrecedingError(); } // decompress if (compression_) { const T* inputPtrHost = nullptr; const T* inputPtrGPU = nullptr; std::tie(inputPtrHost, inputPtrGPU) = translate_gpu_pointer(input); // Add explicit default stream synchronization event_.record(nullptr); event_.stream_wait(stream_.get()); if (inputPtrGPU == nullptr) { // input on HOST gpu::check_status( gpu::memcpy_async(static_cast(freqDomainCompressedDataGPU_.data()), static_cast(inputPtrHost), freqDomainCompressedDataGPU_.size() * sizeof(decltype(*(freqDomainCompressedDataGPU_.data()))), gpu::flag::MemcpyHostToDevice, stream_.get())); compression_->decompress(stream_, freqDomainCompressedDataGPU_.data(), freqDomainDataGPU_); } else { // input on GPU compression_->decompress(stream_, inputPtrGPU, freqDomainDataGPU_); } } // Z if (transformZ_) { zStickSymmetry_->apply(); transformZ_->backward(); } // transpose if (transformZ_) transpose_->pack_backward(); } template auto ExecutionGPU::backward_exchange(const bool nonBlockingExchange) -> void { HOST_TIMING_SCOPED("exchange_start") transpose_->exchange_backward_start(nonBlockingExchange); } template auto ExecutionGPU::backward_xy(const SpfftProcessingUnitType outputLocation) -> void { HOST_TIMING_START("exechange_fininalize"); transpose_->exchange_backward_finalize(); HOST_TIMING_STOP("exechange_fininalize"); if (transformXY_) transpose_->unpack_backward(); // XY if (transformXY_) { planeSymmetry_->apply(); transformXY_->backward(); if (outputLocation & SpfftProcessingUnitType::SPFFT_PU_HOST) { copy_from_gpu_async(stream_, spaceDomainDataExternalGPU_, spaceDomainDataExternalHost_); } } } template auto ExecutionGPU::synchronize() -> void { gpu::stream_synchronize(stream_.get()); } template auto ExecutionGPU::space_domain_data_host() -> HostArrayView3D { return spaceDomainDataExternalHost_; } template auto ExecutionGPU::space_domain_data_gpu() -> GPUArrayView3D { return spaceDomainDataExternalGPU_; } // instatiate templates for float and double template class ExecutionGPU; #ifdef SPFFT_SINGLE_PRECISION template class ExecutionGPU; #endif } // namespace spfft SpFFT-0.9.10/src/execution/execution_gpu.hpp000066400000000000000000000115041357247005600207160ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_EXECUTION_GPU #define SPFFT_EXECUTION_GPU #include #include #include "compression/compression_gpu.hpp" #include "compression/indices.hpp" #include "fft/transform_interface.hpp" #include "gpu_util/gpu_event_handle.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "memory/host_array.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/types.h" #include "symmetry/symmetry.hpp" #include "transpose/transpose.hpp" #include "util/common_types.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #endif namespace spfft { // Controls the execution of the 3D FFT from a compressed format in frequency space and slices in // space domain. Memory is NOT owned by this class and must remain valid during the lifetime. template class ExecutionGPU { public: // Initialize a local execution on GPU ExecutionGPU(const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2, GPUArray::type>& gpuArray1, GPUArray::type>& gpuArray2, const std::shared_ptr>& fftWorkBuffer); #ifdef SPFFT_MPI // Initialize a distributed execution on GPU ExecutionGPU(MPICommunicatorHandle comm, const SpfftExchangeType exchangeType, const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2, GPUArray::type>& gpuArray1, GPUArray::type>& gpuArray2, const std::shared_ptr>& fftWorkBuffer); #endif // transform forward from a given memory location (Host or GPU). // The output is located on the GPU. auto forward_z(T* output, const SpfftScalingType scalingType) -> void; auto forward_exchange(const bool nonBlockingExchange) -> void; auto forward_xy(const SpfftProcessingUnitType inputLocation) -> void; // transform backward into a given memory location (Host or GPU). // The input is taken from the GPU. auto backward_z(const T* input) -> void; auto backward_exchange(const bool nonBlockingExchange) -> void; auto backward_xy(const SpfftProcessingUnitType outputLocation) -> void; auto synchronize() -> void; // The space domain data on Host auto space_domain_data_host() -> HostArrayView3D; // The space domain data on GPU auto space_domain_data_gpu() -> GPUArrayView3D; private: GPUStreamHandle stream_; GPUEventHandle event_; int numThreads_; T scalingFactor_; std::unique_ptr transformZ_; std::unique_ptr transpose_; std::unique_ptr transformXY_; std::unique_ptr zStickSymmetry_; std::unique_ptr planeSymmetry_; std::unique_ptr compression_; HostArrayView3D spaceDomainDataExternalHost_; GPUArrayView3D spaceDomainDataExternalGPU_; GPUArrayView2D::type> freqDomainDataGPU_; GPUArrayView1D freqDomainCompressedDataGPU_; }; } // namespace spfft #endif SpFFT-0.9.10/src/execution/execution_host.cpp000066400000000000000000000415041357247005600210760ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "execution/execution_host.hpp" #include "compression/indices.hpp" #include "fft/transform_1d_host.hpp" #include "fft/transform_real_1d_host.hpp" #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "spfft/exceptions.hpp" #include "symmetry/symmetry_host.hpp" #include "timing/timing.hpp" #include "transpose/transpose_host.hpp" #include "util/common_types.hpp" #ifdef SPFFT_MPI #include "transpose/transpose_mpi_buffered_host.hpp" #include "transpose/transpose_mpi_compact_buffered_host.hpp" #include "transpose/transpose_mpi_unbuffered_host.hpp" #endif namespace spfft { template ExecutionHost::ExecutionHost(const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2) : numThreads_(numThreads), scalingFactor_(static_cast( 1.0 / static_cast(param->dim_x() * param->dim_y() * param->dim_z()))) { HOST_TIMING_SCOPED("Execution init"); const SizeType numLocalZSticks = param->num_z_sticks(0); const SizeType numLocalXYPlanes = param->num_xy_planes(0); std::set uniqueXIndices; for (const auto& xyIndex : param->z_stick_xy_indices(0)) { uniqueXIndices.emplace(static_cast(xyIndex / param->dim_y())); } auto freqDomainZ3D = create_3d_view(array1, 0, 1, numLocalZSticks, param->dim_z()); freqDomainData_ = create_2d_view(freqDomainZ3D, 0, numLocalZSticks, param->dim_z()); auto freqDomainXY = create_3d_view(array2, 0, param->dim_z(), param->dim_x_freq(), param->dim_y()); transpose_.reset(new TransposeHost(param, freqDomainXY, freqDomainData_)); if (param->local_value_indices().size() > 0) { compression_.reset(new CompressionHost(param)); } if (numLocalZSticks > 0) { // Z transformZBackward_.reset(new Transform1DPlanesHost(freqDomainZ3D, freqDomainZ3D, false, false, FFTW_BACKWARD, numThreads)); transformZForward_.reset(new Transform1DPlanesHost(freqDomainZ3D, freqDomainZ3D, false, false, FFTW_FORWARD, numThreads)); } if (numLocalXYPlanes > 0) { // Y transformYBackward_.reset(new Transform1DVerticalHost(freqDomainXY, freqDomainXY, false, false, FFTW_BACKWARD, uniqueXIndices)); transformYForward_.reset(new Transform1DVerticalHost(freqDomainXY, freqDomainXY, false, false, FFTW_FORWARD, uniqueXIndices)); // X if (param->transform_type() == SPFFT_TRANS_R2C) { if (param->zero_zero_stick_index() < param->num_z_sticks(0)) { zStickSymmetry_.reset(new StickSymmetryHost(HostArrayView1D>( &freqDomainData_(param->zero_zero_stick_index(), 0), freqDomainData_.dim_inner(), freqDomainData_.pinned()))); } planeSymmetry_.reset(new PlaneSymmetryHost(freqDomainXY)); spaceDomainDataExternal_ = create_new_type_3d_view(array1, param->dim_z(), param->dim_y(), param->dim_x()); transformXBackward_.reset(new TransformReal1DPlanesHost( freqDomainXY, spaceDomainDataExternal_, true, false, numThreads)); transformXForward_.reset(new TransformReal1DPlanesHost( spaceDomainDataExternal_, freqDomainXY, false, true, numThreads)); } else { zStickSymmetry_.reset(new Symmetry()); planeSymmetry_.reset(new Symmetry()); auto spaceDomainData = create_3d_view(array1, 0, param->dim_z(), param->dim_y(), param->dim_x_freq()); spaceDomainDataExternal_ = create_new_type_3d_view(array1, param->dim_z(), param->dim_y(), 2 * param->dim_x()); transformXBackward_.reset(new Transform1DPlanesHost(freqDomainXY, spaceDomainData, true, false, FFTW_BACKWARD, numThreads)); transformXForward_.reset(new Transform1DPlanesHost(spaceDomainData, freqDomainXY, false, true, FFTW_FORWARD, numThreads)); } } } #ifdef SPFFT_MPI template ExecutionHost::ExecutionHost(MPICommunicatorHandle comm, const SpfftExchangeType exchangeType, const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2) : numThreads_(numThreads), scalingFactor_(static_cast( 1.0 / static_cast(param->dim_x() * param->dim_y() * param->dim_z()))), zStickSymmetry_(new Symmetry()), planeSymmetry_(new Symmetry()) { HOST_TIMING_SCOPED("Execution init"); const SizeType numLocalZSticks = param->num_z_sticks(comm.rank()); const SizeType numLocalXYPlanes = param->num_xy_planes(comm.rank()); // get unique x indices to only compute non-zero y-transforms std::set uniqueXIndices; for (SizeType r = 0; r < comm.size(); ++r) { for (const auto& xyIndex : param->z_stick_xy_indices(r)) { uniqueXIndices.emplace(static_cast(xyIndex / param->dim_y())); } } auto freqDomainZ3D = create_3d_view(array1, 0, 1, numLocalZSticks, param->dim_z()); freqDomainData_ = create_2d_view(freqDomainZ3D, 0, numLocalZSticks, param->dim_z()); auto freqDomainXY = create_3d_view(array2, 0, numLocalXYPlanes, param->dim_x_freq(), param->dim_y()); auto& spaceDomainArray = array1; // create external view with if (param->transform_type() == SPFFT_TRANS_R2C) { spaceDomainDataExternal_ = create_new_type_3d_view(spaceDomainArray, numLocalXYPlanes, param->dim_y(), param->dim_x()); } else { spaceDomainDataExternal_ = create_new_type_3d_view(spaceDomainArray, numLocalXYPlanes, param->dim_y(), 2 * param->dim_x()); } if (param->local_value_indices().size() > 0) { compression_.reset(new CompressionHost(param)); } if (numLocalZSticks > 0) { // apply hermitian symmetry for x=0, y=0 stick if (param->transform_type() == SPFFT_TRANS_R2C && param->zero_zero_stick_index() < freqDomainData_.dim_outer()) { zStickSymmetry_.reset(new StickSymmetryHost( HostArrayView1D>(&freqDomainData_(param->zero_zero_stick_index(), 0), freqDomainData_.dim_inner(), freqDomainData_.pinned()))); } transformZForward_ = std::unique_ptr(new Transform1DPlanesHost( freqDomainZ3D, freqDomainZ3D, false, false, FFTW_FORWARD, numThreads)); transformZBackward_ = std::unique_ptr(new Transform1DPlanesHost( freqDomainZ3D, freqDomainZ3D, false, false, FFTW_BACKWARD, numThreads)); } if (numLocalXYPlanes > 0) { transformYBackward_.reset(new Transform1DVerticalHost(freqDomainXY, freqDomainXY, false, false, FFTW_BACKWARD, uniqueXIndices)); transformYForward_.reset(new Transform1DVerticalHost(freqDomainXY, freqDomainXY, false, false, FFTW_FORWARD, uniqueXIndices)); if (param->transform_type() == SPFFT_TRANS_R2C) { transformXBackward_.reset(new TransformReal1DPlanesHost( freqDomainXY, spaceDomainDataExternal_, true, false, numThreads)); transformXForward_.reset(new TransformReal1DPlanesHost( spaceDomainDataExternal_, freqDomainXY, false, true, numThreads)); planeSymmetry_.reset(new PlaneSymmetryHost(freqDomainXY)); } else { auto spaceDomainData = create_3d_view(spaceDomainArray, 0, numLocalXYPlanes, param->dim_y(), param->dim_x()); transformXBackward_.reset(new Transform1DPlanesHost(freqDomainXY, spaceDomainData, true, false, FFTW_BACKWARD, numThreads)); transformXForward_.reset(new Transform1DPlanesHost(spaceDomainData, freqDomainXY, false, true, FFTW_FORWARD, numThreads)); } } switch (exchangeType) { case SpfftExchangeType::SPFFT_EXCH_UNBUFFERED: { transpose_.reset( new TransposeMPIUnbufferedHost(param, comm, freqDomainXY, freqDomainData_)); } break; case SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED: { auto transposeBufferZ = create_1d_view( array2, 0, param->total_num_xy_planes() * param->num_z_sticks(comm.rank())); auto transposeBufferXY = create_1d_view( array1, 0, param->total_num_z_sticks() * param->num_xy_planes(comm.rank())); transpose_.reset(new TransposeMPICompactBufferedHost( param, comm, freqDomainXY, freqDomainData_, transposeBufferXY, transposeBufferZ)); } break; case SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED_FLOAT: { auto transposeBufferZ = create_1d_view( array2, 0, param->total_num_xy_planes() * param->num_z_sticks(comm.rank())); auto transposeBufferXY = create_1d_view( array1, 0, param->total_num_z_sticks() * param->num_xy_planes(comm.rank())); transpose_.reset(new TransposeMPICompactBufferedHost( param, comm, freqDomainXY, freqDomainData_, transposeBufferXY, transposeBufferZ)); } break; case SpfftExchangeType::SPFFT_EXCH_BUFFERED: { auto transposeBufferZ = create_1d_view( array2, 0, param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size()); auto transposeBufferXY = create_1d_view( array1, 0, param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size()); transpose_.reset(new TransposeMPIBufferedHost( param, comm, freqDomainXY, freqDomainData_, transposeBufferXY, transposeBufferZ)); } break; case SpfftExchangeType::SPFFT_EXCH_BUFFERED_FLOAT: { auto transposeBufferZ = create_1d_view( array2, 0, param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size()); auto transposeBufferXY = create_1d_view( array1, 0, param->max_num_z_sticks() * param->max_num_xy_planes() * comm.size()); transpose_.reset(new TransposeMPIBufferedHost( param, comm, freqDomainXY, freqDomainData_, transposeBufferXY, transposeBufferZ)); } break; default: throw InvalidParameterError(); } } #endif template auto ExecutionHost::forward_xy() -> void { SPFFT_OMP_PRAGMA("omp parallel num_threads(numThreads_)") { SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("x transform"); } if (transformXForward_) transformXForward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("x transform"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("y transform"); } if (transformYForward_) transformYForward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("y transform"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("pack"); } if (transformYForward_) transpose_->pack_forward(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("pack"); } } } template auto ExecutionHost::forward_exchange(const bool nonBlockingExchange) -> void { HOST_TIMING_SCOPED("exchange_start") // must be called outside omp parallel region (MPI restriction on thread id) transpose_->exchange_forward_start(nonBlockingExchange); // SPFFT_OMP_PRAGMA("omp barrier") // ensure exchange is done } template auto ExecutionHost::forward_z(T* output, const SpfftScalingType scalingType) -> void { // must be called outside omp parallel region (MPI restriction on thread id) HOST_TIMING_START("exechange_fininalize"); transpose_->exchange_forward_finalize(); HOST_TIMING_STOP("exechange_fininalize"); HOST_TIMING_STOP("exchange") SPFFT_OMP_PRAGMA("omp parallel num_threads(numThreads_)") { SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("unpack"); } if (transformZForward_) transpose_->unpack_forward(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("unpack"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("z transform"); } if (transformZForward_) transformZForward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("z transform"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("compression"); } if (compression_) compression_->compress(freqDomainData_, output, scalingType == SpfftScalingType::SPFFT_FULL_SCALING, scalingFactor_); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("compression"); } } } template auto ExecutionHost::backward_z(const T* input) -> void { SPFFT_OMP_PRAGMA("omp parallel num_threads(numThreads_)") { SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("compression"); } if (compression_) compression_->decompress(input, freqDomainData_); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("compression"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("z symmetrization"); } zStickSymmetry_->apply(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("z symmetrization"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("z transform"); } if (transformZBackward_) transformZBackward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("z transform"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("pack"); } if (transformZBackward_) transpose_->pack_backward(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("pack"); } } } template auto ExecutionHost::backward_exchange(const bool nonBlockingExchange) -> void { HOST_TIMING_SCOPED("exchange_start") // must be called outside omp parallel region (MPI restriction on thread id) transpose_->exchange_backward_start(nonBlockingExchange); } template auto ExecutionHost::backward_xy() -> void { // must be called outside omp parallel region (MPI restriction on thread id) HOST_TIMING_START("exechange_fininalize"); transpose_->exchange_forward_finalize(); HOST_TIMING_STOP("exechange_fininalize"); HOST_TIMING_STOP("exchange") SPFFT_OMP_PRAGMA("omp parallel num_threads(numThreads_)") { SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("unpack"); } if (transformYBackward_) transpose_->unpack_backward(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("unpack"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("xy symmetrization"); } planeSymmetry_->apply(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("xy symmetrization"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("y transform"); } if (transformYBackward_) transformYBackward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("y transform"); } SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_START("x transform"); } if (transformXBackward_) transformXBackward_->execute(); SPFFT_OMP_PRAGMA("omp master") { HOST_TIMING_STOP("x transform"); } } } template auto ExecutionHost::space_domain_data() -> HostArrayView3D { return spaceDomainDataExternal_; } // instatiate templates for float and double template class ExecutionHost; #ifdef SPFFT_SINGLE_PRECISION template class ExecutionHost; #endif } // namespace spfft SpFFT-0.9.10/src/execution/execution_host.hpp000066400000000000000000000100061357247005600210740ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_EXECUTION_HOST_HPP #define SPFFT_EXECUTION_HOST_HPP #include #include #include #include "compression/compression_host.hpp" #include "compression/indices.hpp" #include "fft/transform_interface.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/types.h" #include "symmetry/symmetry.hpp" #include "timing/timing.hpp" #include "transpose/transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_init_handle.hpp" #endif namespace spfft { // Controls the execution of the 3D FFT from a compressed format in frequency space and slices in // space domain. Memory is NOT owned by this class and must remain valid during the lifetime. template class ExecutionHost { public: // Initialize a local execution on Host ExecutionHost(const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2); #ifdef SPFFT_MPI // Initialize a distributed execution on Host ExecutionHost(MPICommunicatorHandle comm, const SpfftExchangeType exchangeType, const int numThreads, std::shared_ptr param, HostArray>& array1, HostArray>& array2); #endif // Transform forward auto forward_z(T* output, const SpfftScalingType scalingType) -> void; auto forward_exchange(const bool nonBlockingExchange) -> void; auto forward_xy() -> void; // Transform backward auto backward_z(const T* input) -> void; auto backward_exchange(const bool nonBlockingExchange) -> void; auto backward_xy() -> void; // Access the space domain data auto space_domain_data() -> HostArrayView3D; private: int numThreads_; T scalingFactor_; std::unique_ptr transformZBackward_; std::unique_ptr transformZForward_; std::unique_ptr transformYBackward_; std::unique_ptr transformYForward_; std::unique_ptr transformXBackward_; std::unique_ptr transformXForward_; std::unique_ptr transpose_; std::unique_ptr zStickSymmetry_; std::unique_ptr planeSymmetry_; std::unique_ptr compression_; HostArrayView3D spaceDomainDataExternal_; HostArrayView2D> freqDomainData_; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/000077500000000000000000000000001357247005600141025ustar00rootroot00000000000000SpFFT-0.9.10/src/fft/fftw_plan_1d.hpp000066400000000000000000000216661357247005600171720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_FFTW_PLAN_HPP #define SPFFT_FFTW_PLAN_HPP #include #include #include #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" namespace spfft { template class FFTWPlan; template <> class FFTWPlan { public: using ComplexType = std::complex; // Create standard 1d fftw plan. // If input and output pointers are equal, in-place transform is created. FFTWPlan(ComplexType* input, ComplexType* output, const SizeType size, const int sign) : plan_(nullptr), size_(size), inPlace_(input == output), alignmentInput_(fftw_alignment_of(reinterpret_cast(input))), alignmentOutput_(fftw_alignment_of(reinterpret_cast(output))) { auto flags = FFTW_ESTIMATE; if (input != output) { flags = flags | FFTW_DESTROY_INPUT; // allow input override for out-of-place transform } plan_ = fftw_plan_dft_1d(size, reinterpret_cast(input), reinterpret_cast(output), sign, flags); if (!plan_) throw FFTWError(); } // Create strided 1d fftw plan. // If input and output pointers are equal, in-place transform is created. FFTWPlan(ComplexType* input, ComplexType* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany, const int sign) { int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; auto flags = FFTW_ESTIMATE; if (input != output) { flags = flags | FFTW_DESTROY_INPUT; // allow input override for out-of-place transform } plan_ = fftw_plan_many_dft(rank, n, (int)howmany, reinterpret_cast(input), inembed, (int)istride, (int)idist, reinterpret_cast(output), onembed, (int)ostride, (int)odist, sign, flags); if (!plan_) throw FFTWError(); } FFTWPlan(const FFTWPlan& other) = delete; FFTWPlan(FFTWPlan&& other) noexcept { if (plan_) fftw_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; } auto operator=(const FFTWPlan& other) -> FFTWPlan& = delete; auto operator=(FFTWPlan&& other) noexcept -> FFTWPlan& { if (plan_) fftw_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; return *this; } // Get plan handle inline auto get() -> fftw_plan { return plan_; }; // Release ownership of plan handle inline auto release() -> fftw_plan { fftw_plan planLocal = plan_; plan_ = nullptr; return planLocal; }; inline auto empty() const noexcept -> bool { return !plan_; } inline auto size() const noexcept -> SizeType { return size_; } // Plan created with in-place transform inline auto in_place() const noexcept -> bool { return inPlace_; } // Execute on input / output provided to constructor. // Undefinded behaviour if empty(). auto execute() -> void { fftw_execute(plan_); } // Execute on given input / output. // The alignment of input and output must match the pointers given to the constructor. // If the plan was not setup for in-place transforms, input and output must not be equal // Undefinded behaviour if empty(). auto execute(ComplexType* input, ComplexType* output) -> void { assert(inPlace_ == (input == output)); assert(fftw_alignment_of(reinterpret_cast(input)) == alignmentInput_); assert(fftw_alignment_of(reinterpret_cast(output)) == alignmentOutput_); fftw_execute_dft(plan_, reinterpret_cast(input), reinterpret_cast(output)); } ~FFTWPlan() { if (plan_) fftw_destroy_plan(plan_); plan_ = nullptr; } private: fftw_plan plan_ = nullptr; SizeType size_ = 0; bool inPlace_ = false; int alignmentInput_ = 0; int alignmentOutput_ = 0; }; #ifdef SPFFT_SINGLE_PRECISION template <> class FFTWPlan { public: using ComplexType = std::complex; // Create standard 1d fftw plan. // If input and output pointers are equal, in-place transform is created. FFTWPlan(ComplexType* input, ComplexType* output, const SizeType size, const int sign) : plan_(nullptr), size_(size), inPlace_(input == output), alignmentInput_(fftwf_alignment_of(reinterpret_cast(input))), alignmentOutput_(fftwf_alignment_of(reinterpret_cast(output))) { plan_ = fftwf_plan_dft_1d(size, reinterpret_cast(input), reinterpret_cast(output), sign, FFTW_ESTIMATE); if (!plan_) throw FFTWError(); } // Create strided 1d fftw plan. // If input and output pointers are equal, in-place transform is created. FFTWPlan(ComplexType* input, ComplexType* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany, const int sign) { int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; plan_ = fftwf_plan_many_dft(rank, n, (int)howmany, reinterpret_cast(input), inembed, (int)istride, (int)idist, reinterpret_cast(output), onembed, (int)ostride, (int)odist, sign, FFTW_ESTIMATE); if (!plan_) throw FFTWError(); } FFTWPlan(const FFTWPlan& other) = delete; FFTWPlan(FFTWPlan&& other) noexcept { if (plan_) fftwf_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; } auto operator=(const FFTWPlan& other) -> FFTWPlan& = delete; auto operator=(FFTWPlan&& other) noexcept -> FFTWPlan& { if (plan_) fftwf_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; return *this; } // Get plan handle inline auto get() -> fftwf_plan { return plan_; }; // Release ownership of plan handle inline auto release() -> fftwf_plan { fftwf_plan planLocal = plan_; plan_ = nullptr; return planLocal; }; inline auto empty() const noexcept -> bool { return !plan_; } inline auto size() const noexcept -> SizeType { return size_; } // Plan created with in-place transform inline auto in_place() const noexcept -> bool { return inPlace_; } // Execute on input / output provided to constructor. // Undefinded behaviour if empty(). auto execute() -> void { fftwf_execute(plan_); } // Execute on given input / output. // The alignment of input and output must match the pointers given to the constructor. // If the plan was not setup for in-place transforms, input and output must not be equal // Undefinded behaviour if empty(). auto execute(ComplexType* input, ComplexType* output) -> void { assert(inPlace_ == (input == output)); assert(fftwf_alignment_of(reinterpret_cast(input)) == alignmentInput_); assert(fftwf_alignment_of(reinterpret_cast(output)) == alignmentOutput_); fftwf_execute_dft(plan_, reinterpret_cast(input), reinterpret_cast(output)); } ~FFTWPlan() { if (plan_) fftwf_destroy_plan(plan_); plan_ = nullptr; } private: fftwf_plan plan_ = nullptr; SizeType size_ = 0; bool inPlace_ = false; int alignmentInput_ = 0; int alignmentOutput_ = 0; }; #endif } // namespace spfft #endif SpFFT-0.9.10/src/fft/fftw_real_plan_1d.hpp000066400000000000000000000207451357247005600201720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_FFTW_REAL_PLAN_HPP #define SPFFT_FFTW_REAL_PLAN_HPP #include #include #include #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" namespace spfft { template class FFTWRealPlan; template <> class FFTWRealPlan { public: using ComplexType = std::complex; // Create strided 1d fftw real plan r2c FFTWRealPlan(double* input, ComplexType* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany) { assert(reinterpret_cast(input) != reinterpret_cast(output)); // must not be in place int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; auto flags = FFTW_ESTIMATE | FFTW_DESTROY_INPUT; plan_ = fftw_plan_many_dft_r2c(rank, n, (int)howmany, input, inembed, (int)istride, (int)idist, reinterpret_cast(output), onembed, (int)ostride, (int)odist, flags); if (!plan_) throw FFTWError(); } // Create strided 1d fftw real plan c2r FFTWRealPlan(ComplexType* input, double* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany) { assert(reinterpret_cast(input) != reinterpret_cast(output)); // must not be in place int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; auto flags = FFTW_ESTIMATE | FFTW_DESTROY_INPUT; plan_ = fftw_plan_many_dft_c2r(rank, n, (int)howmany, reinterpret_cast(input), inembed, (int)istride, (int)idist, output, onembed, (int)ostride, (int)odist, flags); if (!plan_) throw FFTWError(); } FFTWRealPlan(const FFTWRealPlan& other) = delete; FFTWRealPlan(FFTWRealPlan&& other) noexcept { if (plan_) fftw_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; } auto operator=(const FFTWRealPlan& other) -> FFTWRealPlan& = delete; auto operator=(FFTWRealPlan&& other) noexcept -> FFTWRealPlan& { if (plan_) fftw_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; return *this; } // Get plan handle inline auto get() -> fftw_plan { return plan_; }; // Release ownership of plan handle inline auto release() -> fftw_plan { fftw_plan planLocal = plan_; plan_ = nullptr; return planLocal; }; inline auto empty() const noexcept -> bool { return !plan_; } inline auto size() const noexcept -> SizeType { return size_; } // Plan created with in-place transform inline auto in_place() const noexcept -> bool { return inPlace_; } // Execute on input / output provided to constructor. // Undefinded behaviour if empty(). auto execute() -> void { fftw_execute(plan_); } ~FFTWRealPlan() { if (plan_) fftw_destroy_plan(plan_); plan_ = nullptr; } private: fftw_plan plan_ = nullptr; SizeType size_ = 0; bool inPlace_ = false; }; #ifdef SPFFT_SINGLE_PRECISION template <> class FFTWRealPlan { public: using ComplexType = std::complex; // Create strided 1d fftwf real plan r2c FFTWRealPlan(float* input, ComplexType* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany) { assert(reinterpret_cast(input) != reinterpret_cast(output)); // must not be in place int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; auto flags = FFTW_ESTIMATE | FFTW_DESTROY_INPUT; plan_ = fftwf_plan_many_dft_r2c(rank, n, (int)howmany, input, inembed, (int)istride, (int)idist, reinterpret_cast(output), onembed, (int)ostride, (int)odist, flags); if (!plan_) throw FFTWError(); } // Create strided 1d fftwf real plan c2r FFTWRealPlan(ComplexType* input, float* output, const SizeType size, const SizeType istride, const SizeType ostride, const SizeType idist, const SizeType odist, const SizeType howmany) { assert(reinterpret_cast(input) != reinterpret_cast(output)); // must not be in place int rank = 1; int n[] = {(int)size}; int inembed[] = {n[0]}; int onembed[] = {n[0]}; auto flags = FFTW_ESTIMATE | FFTW_DESTROY_INPUT; plan_ = fftwf_plan_many_dft_c2r(rank, n, (int)howmany, reinterpret_cast(input), inembed, (int)istride, (int)idist, output, onembed, (int)ostride, (int)odist, flags); if (!plan_) throw FFTWError(); } FFTWRealPlan(const FFTWRealPlan& other) = delete; FFTWRealPlan(FFTWRealPlan&& other) noexcept { if (plan_) fftwf_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; } auto operator=(const FFTWRealPlan& other) -> FFTWRealPlan& = delete; auto operator=(FFTWRealPlan&& other) noexcept -> FFTWRealPlan& { if (plan_) fftwf_destroy_plan(plan_); plan_ = other.plan_; other.plan_ = nullptr; return *this; } // Get plan handle inline auto get() -> fftwf_plan { return plan_; }; // Release ownership of plan handle inline auto release() -> fftwf_plan { fftwf_plan planLocal = plan_; plan_ = nullptr; return planLocal; }; inline auto empty() const noexcept -> bool { return !plan_; } inline auto size() const noexcept -> SizeType { return size_; } // Plan created with in-place transform inline auto in_place() const noexcept -> bool { return inPlace_; } // Execute on input / output provided to constructor. // Undefinded behaviour if empty(). auto execute() -> void { fftwf_execute(plan_); } // Execute on given input / output. // The alignment of input and output must match the pointers given to the constructor. // If the plan was not setup for in-place transforms, input and output must not be equal // Undefinded behaviour if empty(). auto execute(ComplexType* input, ComplexType* output) -> void { assert(inPlace_ == (input == output)); assert(fftwf_alignment_of(reinterpret_cast(input)) == alignmentInput_); assert(fftwf_alignment_of(reinterpret_cast(output)) == alignmentOutput_); fftwf_execute_dft(plan_, reinterpret_cast(input), reinterpret_cast(output)); } ~FFTWRealPlan() { if (plan_) fftwf_destroy_plan(plan_); plan_ = nullptr; } private: fftwf_plan plan_ = nullptr; SizeType size_ = 0; bool inPlace_ = false; int alignmentInput_ = 0; int alignmentOutput_ = 0; }; #endif } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_1d_gpu.hpp000066400000000000000000000113641357247005600200720ustar00rootroot00000000000000 /* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_1D_GPU_HPP #define SPFFT_TRANSFORM_1D_GPU_HPP #include #include #include #include #include "fft/transform_interface.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_runtime_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class Transform1DGPU : public TransformGPU { public: using ValueType = T; using ComplexType = gpu::fft::ComplexType; Transform1DGPU(GPUArrayView2D::type>& data, GPUStreamHandle stream, std::shared_ptr> workBuffer) : stream_(std::move(stream)), workBuffer_(std::move(workBuffer)), dataPtr_(data.data()) { assert(workBuffer_); std::size_t worksize = 0; int rank = 1; int n[1] = {data.dim_inner()}; int nembed[1] = {data.dim_inner()}; int stride = 1; int dist = data.dim_inner(); int batch = data.dim_outer(); // create plan gpu::fft::check_result(gpu::fft::create(&plan_)); gpu::fft::check_result(gpu::fft::set_auto_allocation(plan_, 0)); gpu::fft::check_result(gpu::fft::make_plan_many( plan_, rank, n, nembed, stride, dist, nembed, stride, dist, gpu::fft::TransformType::ComplexToComplex::value, batch, &worksize)); // set stream gpu::fft::check_result(gpu::fft::set_stream(plan_, stream_.get())); // resize work buffer if necessary if (workBuffer_->size() < worksize) { *workBuffer_ = GPUArray(worksize); } } Transform1DGPU(const Transform1DGPU& transform) = delete; Transform1DGPU(Transform1DGPU&& transform) noexcept : stream_(std::move(transform.stream_)), plan_(std::move(transform.plan_)), workBuffer_(std::move(transform.workBuffer_)), dataPtr_(transform.dataPtr_) { transform.plan_ = 0; } ~Transform1DGPU() { if (plan_) { gpu::fft::destroy(plan_); } } auto operator=(const Transform1DGPU& transform) -> Transform1DGPU& = delete; auto operator=(Transform1DGPU&& transform) noexcept -> Transform1DGPU& { if (plan_) { gpu::fft::destroy(plan_); } stream_ = std::move(transform.stream_); plan_ = std::move(transform.plan_); workBuffer_ = std::move(transform.workBuffer_); dataPtr_ = transform.dataPtr_; transform.plan_ = 0; return *this; } inline auto device_id() const noexcept -> int { return stream_.device_id(); } auto forward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(plan_, workBuffer_->data())); gpu::fft::check_result( gpu::fft::execute(plan_, dataPtr_, dataPtr_, gpu::fft::TransformDirection::Forward)); } auto backward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(plan_, workBuffer_->data())); gpu::fft::check_result( gpu::fft::execute(plan_, dataPtr_, dataPtr_, gpu::fft::TransformDirection::Backward)); } private: GPUStreamHandle stream_; gpu::fft::HandleType plan_ = 0; std::shared_ptr> workBuffer_; typename gpu::fft::ComplexType::type* dataPtr_; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_1d_host.hpp000066400000000000000000000220421357247005600202470ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_1D_HOST_HPP #define SPFFT_TRANSFORM_1D_HOST_HPP #include #include #include #include #include "fft/fftw_plan_1d.hpp" #include "fft/transform_interface.hpp" #include "memory/host_array_view.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" namespace spfft { // Computes the FFT in 1D along either the innermost dimension (not transposed) or the second // innermost dimension (transposed) // The transforms are computed in batches aligned to inner 2d planes template class Transform1DPlanesHost : public TransformHost { public: static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; Transform1DPlanesHost(HostArrayView3D inputData, HostArrayView3D outputData, bool transposeInputData, bool transposeOutputData, int sign, int maxNumThreads) { assert(inputData.dim_outer() == outputData.dim_outer()); // only one is transposed assert((transposeInputData != transposeOutputData) || (inputData.dim_inner() == outputData.dim_inner())); assert((transposeInputData != transposeOutputData) || (inputData.dim_mid() == outputData.dim_mid())); // none or both transposed assert((transposeInputData == transposeOutputData) || (inputData.dim_inner() == outputData.dim_mid())); assert((transposeInputData == transposeOutputData) || (inputData.dim_mid() == outputData.dim_inner())); // transposed case must not be in-place assert(!(inputData.data() == outputData.data() && (transposeInputData || transposeOutputData))); // make sure maxNumThreads is at least 1 SizeType numSplitsPerPlane = maxNumThreads < 1 ? 1 : maxNumThreads; // only use at most as many splits as required to create work for every thread if (numSplitsPerPlane > 1 && inputData.dim_outer() > numSplitsPerPlane) { numSplitsPerPlane = 2; } const SizeType numTransformsPerPlane = transposeInputData ? inputData.dim_inner() : inputData.dim_mid(); // make sure there are at most as many splits as transforms per plane numSplitsPerPlane = numTransformsPerPlane < numSplitsPerPlane ? numTransformsPerPlane : numSplitsPerPlane; // set fftw plan parameters const SizeType size = transposeInputData ? inputData.dim_mid() : inputData.dim_inner(); const SizeType inputStride = transposeInputData ? inputData.dim_inner() : 1; const SizeType outputStride = transposeOutputData ? outputData.dim_inner() : 1; const SizeType inputDist = transposeInputData ? 1 : inputData.dim_inner(); const SizeType outputDist = transposeOutputData ? 1 : outputData.dim_inner(); const SizeType numTransformsPerSplit = numTransformsPerPlane / numSplitsPerPlane; const SizeType inputSplitStrideMid = transposeInputData ? 0 : numTransformsPerSplit; const SizeType inputSplitStrideInner = transposeInputData ? numTransformsPerSplit : 0; const SizeType outputSplitStrideMid = transposeOutputData ? 0 : numTransformsPerSplit; const SizeType outputSplitStrideInner = transposeOutputData ? numTransformsPerSplit : 0; // determine number of transforms per plane // create plans within each plane transforms_.reserve(inputData.dim_outer() * numSplitsPerPlane); for (SizeType idxOuter = 0; idxOuter < inputData.dim_outer(); ++idxOuter) { for (SizeType idxSplit = 0; idxSplit < numSplitsPerPlane; ++idxSplit) { const SizeType howmany = idxSplit == numSplitsPerPlane - 1 ? numTransformsPerSplit + numTransformsPerPlane % numSplitsPerPlane : numTransformsPerSplit; transforms_.emplace_back(&(inputData(idxOuter, idxSplit * inputSplitStrideMid, idxSplit * inputSplitStrideInner)), &(outputData(idxOuter, idxSplit * outputSplitStrideMid, idxSplit * outputSplitStrideInner)), size, inputStride, outputStride, inputDist, outputDist, howmany, sign); } } } auto execute() -> void override { SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < transforms_.size(); ++i) { transforms_[i].execute(); } } private: std::vector> transforms_; }; // Computes the FFT in 1D along either the innermost dimension (not transposed) or the second // innermost dimension (transposed). // The transforms are computed in batches aligned to the outer and transform dimension. // The indices of transforms to be computed per plane can be provided as well. template class Transform1DVerticalHost : public TransformHost { public: static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; Transform1DVerticalHost(HostArrayView3D inputData, HostArrayView3D outputData, bool transposeInputData, bool transposeOutputData, int sign, const std::set& inputMidIndices) { assert(inputData.dim_outer() == outputData.dim_outer()); // check case where only one is transposed assert((transposeInputData != transposeOutputData) || (inputData.dim_inner() == outputData.dim_inner())); assert((transposeInputData != transposeOutputData) || (inputData.dim_mid() == outputData.dim_mid())); // none or both transposed assert((transposeInputData == transposeOutputData) || (inputData.dim_inner() == outputData.dim_mid())); assert((transposeInputData == transposeOutputData) || (inputData.dim_mid() == outputData.dim_inner())); // transposed case must not be in-place assert(!(inputData.data() == outputData.data() && (transposeInputData || transposeOutputData))); // set fftw plan parameters const SizeType size = transposeInputData ? inputData.dim_mid() : inputData.dim_inner(); const SizeType inputStride = transposeInputData ? inputData.dim_inner() : 1; const SizeType outputStride = transposeOutputData ? outputData.dim_inner() : 1; const SizeType inputDist = inputData.dim_mid() * inputData.dim_inner(); const SizeType outputDist = outputData.dim_mid() * outputData.dim_inner(); const SizeType howmany = inputData.dim_outer(); // determine number of transforms per plane // create plans within each plane transforms_.reserve(inputMidIndices.size()); for (const auto& midIndex : inputMidIndices) { const SizeType idxMidInput = transposeInputData ? 0 : midIndex; const SizeType idxInnerInput = transposeInputData ? midIndex : 0; const SizeType idxMidOutput = transposeOutputData ? 0 : midIndex; const SizeType idxInnerOutput = transposeOutputData ? midIndex : 0; transforms_.emplace_back(&(inputData(0, idxMidInput, idxInnerInput)), &(outputData(0, idxMidOutput, idxInnerOutput)), size, inputStride, outputStride, inputDist, outputDist, howmany, sign); } } auto execute() -> void override { SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < transforms_.size(); ++i) { transforms_[i].execute(); } } private: std::vector> transforms_; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_2d_gpu.hpp000066400000000000000000000114441357247005600200720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_2D_GPU_HPP #define SPFFT_TRANSFORM_2D_GPU_HPP #include #include #include #include #include "fft/transform_interface.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_runtime_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class Transform2DGPU : public TransformGPU { public: using ValueType = T; using ComplexType = gpu::fft::ComplexType; Transform2DGPU(GPUArrayView3D::type>& data, GPUStreamHandle stream, std::shared_ptr> workBuffer) : stream_(std::move(stream)), workBuffer_(std::move(workBuffer)), dataPtr_(data.data()) { assert(workBuffer_); std::size_t worksize = 0; int rank = 2; int n[2] = {data.dim_mid(), data.dim_inner()}; int nembed[2] = {data.dim_mid(), data.dim_inner()}; int stride = 1; int dist = data.dim_inner() * data.dim_mid(); int batch = data.dim_outer(); // create plan gpu::fft::check_result(gpu::fft::create(&plan_)); gpu::fft::check_result(gpu::fft::set_auto_allocation(plan_, 0)); gpu::fft::check_result(gpu::fft::make_plan_many( plan_, rank, n, nembed, stride, dist, nembed, stride, dist, gpu::fft::TransformType::ComplexToComplex::value, batch, &worksize)); // set stream gpu::fft::check_result(gpu::fft::set_stream(plan_, stream_.get())); // resize work buffer if necessary if (workBuffer_->size() < worksize) { *workBuffer_ = GPUArray(worksize); } } Transform2DGPU(const Transform2DGPU& transform) = delete; Transform2DGPU(Transform2DGPU&& transform) noexcept : stream_(std::move(transform.stream_)), plan_(std::move(transform.plan_)), workBuffer_(std::move(transform.workBuffer_)), dataPtr_(transform.dataPtr_) { transform.plan_ = 0; } ~Transform2DGPU() { if (plan_) { gpu::fft::destroy(plan_); } } auto operator=(const Transform2DGPU& transform) -> Transform2DGPU& = delete; auto operator=(Transform2DGPU&& transform) noexcept -> Transform2DGPU& { if (plan_) { gpu::fft::destroy(plan_); } stream_ = std::move(transform.stream_); plan_ = std::move(transform.plan_); workBuffer_ = std::move(transform.workBuffer_); dataPtr_ = transform.dataPtr_; transform.plan_ = 0; return *this; } inline auto device_id() const noexcept -> int { return stream_.device_id(); } auto forward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(plan_, workBuffer_->data())); gpu::fft::check_result( gpu::fft::execute(plan_, dataPtr_, dataPtr_, gpu::fft::TransformDirection::Forward)); } auto backward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(plan_, workBuffer_->data())); gpu::fft::check_result( gpu::fft::execute(plan_, dataPtr_, dataPtr_, gpu::fft::TransformDirection::Backward)); } private: GPUStreamHandle stream_; gpu::fft::HandleType plan_ = 0; std::shared_ptr> workBuffer_; typename gpu::fft::ComplexType::type* dataPtr_; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_interface.hpp000066400000000000000000000036651357247005600206600ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_INTERFACE_HPP #define SPFFT_TRANSFORM_INTERFACE_HPP #include "spfft/config.h" namespace spfft { class TransformHost { public: virtual auto execute() -> void = 0; virtual ~TransformHost() = default; }; class TransformGPU { public: virtual auto forward() -> void = 0; virtual auto backward() -> void = 0; virtual ~TransformGPU() = default; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_real_1d_host.hpp000066400000000000000000000212301357247005600212500ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_REAL_1D_HOST_HPP #define SPFFT_TRANSFORM_REAL_1D_HOST_HPP #include #include #include #include #include "fft/fftw_real_plan_1d.hpp" #include "fft/transform_interface.hpp" #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" namespace spfft { // Computes the FFT in 1D along either the innermost dimension (not transposed) or the second // innermost dimension (transposed) // The transforms are computed in batches aligned to inner 2d planes template class TransformReal1DPlanesHost : public TransformHost { public: static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; // r2c TransformReal1DPlanesHost(HostArrayView3D inputData, HostArrayView3D outputData, bool transposeInputData, bool transposeOutputData, int maxNumThreads) { assert(inputData.dim_outer() == outputData.dim_outer()); assert(disjoint(inputData, outputData)); // set fftw plan parameters const SizeType size = transposeInputData ? inputData.dim_mid() : inputData.dim_inner(); const SizeType inputStride = transposeInputData ? inputData.dim_inner() : 1; const SizeType outputStride = transposeOutputData ? outputData.dim_inner() : 1; const SizeType inputDist = transposeInputData ? 1 : inputData.dim_inner(); const SizeType outputDist = transposeOutputData ? 1 : outputData.dim_inner(); // make sure maxNumThreads is at least 1 SizeType numSplitsPerPlane = maxNumThreads < 1 ? 1 : maxNumThreads; // only use at most as many splits as required to create work for every thread if (numSplitsPerPlane > 1 && inputData.dim_outer() > numSplitsPerPlane) { numSplitsPerPlane = 2; } const SizeType numTransformsPerPlane = transposeInputData ? inputData.dim_inner() : inputData.dim_mid(); // make sure there are at most as many splits as transforms per plane numSplitsPerPlane = numTransformsPerPlane < numSplitsPerPlane ? numTransformsPerPlane : numSplitsPerPlane; const SizeType numTransformsPerSplit = numTransformsPerPlane / numSplitsPerPlane; const SizeType inputSplitStrideMid = transposeInputData ? 0 : numTransformsPerSplit; const SizeType inputSplitStrideInner = transposeInputData ? numTransformsPerSplit : 0; const SizeType outputSplitStrideMid = transposeOutputData ? 0 : numTransformsPerSplit; const SizeType outputSplitStrideInner = transposeOutputData ? numTransformsPerSplit : 0; // check for non-transposed output assert((transposeOutputData) || (size / 2 + 1 == outputData.dim_inner())); // check for transposed output assert((!transposeOutputData) || (size / 2 + 1 == outputData.dim_mid())); // determine number of transforms per plane // create plans within each plane transforms_.reserve(inputData.dim_outer() * numSplitsPerPlane); for (SizeType idxOuter = 0; idxOuter < inputData.dim_outer(); ++idxOuter) { for (SizeType idxSplit = 0; idxSplit < numSplitsPerPlane; ++idxSplit) { const SizeType howmany = idxSplit == numSplitsPerPlane - 1 ? numTransformsPerSplit + numTransformsPerPlane % numSplitsPerPlane : numTransformsPerSplit; transforms_.emplace_back(&(inputData(idxOuter, idxSplit * inputSplitStrideMid, idxSplit * inputSplitStrideInner)), &(outputData(idxOuter, idxSplit * outputSplitStrideMid, idxSplit * outputSplitStrideInner)), size, inputStride, outputStride, inputDist, outputDist, howmany); } } } // c2r TransformReal1DPlanesHost(HostArrayView3D inputData, HostArrayView3D outputData, bool transposeInputData, bool transposeOutputData, int maxNumThreads) { assert(inputData.dim_outer() == outputData.dim_outer()); assert(disjoint(inputData, outputData)); // set fftw plan parameters const SizeType size = transposeOutputData ? outputData.dim_mid() : outputData.dim_inner(); const SizeType inputStride = transposeInputData ? inputData.dim_inner() : 1; const SizeType outputStride = transposeOutputData ? outputData.dim_inner() : 1; const SizeType inputDist = transposeInputData ? 1 : inputData.dim_inner(); const SizeType outputDist = transposeOutputData ? 1 : outputData.dim_inner(); // make sure maxNumThreads is at least 1 SizeType numSplitsPerPlane = maxNumThreads < 1 ? 1 : maxNumThreads; // only use at most as many splits as required to create work for every thread if (numSplitsPerPlane > 1 && inputData.dim_outer() > numSplitsPerPlane) { numSplitsPerPlane = 2; } const SizeType numTransformsPerPlane = transposeInputData ? inputData.dim_inner() : inputData.dim_mid(); // make sure there are at most as many splits as transforms per plane numSplitsPerPlane = numTransformsPerPlane < numSplitsPerPlane ? numTransformsPerPlane : numSplitsPerPlane; const SizeType numTransformsPerSplit = numTransformsPerPlane / numSplitsPerPlane; const SizeType inputSplitStrideMid = transposeInputData ? 0 : numTransformsPerSplit; const SizeType inputSplitStrideInner = transposeInputData ? numTransformsPerSplit : 0; const SizeType outputSplitStrideMid = transposeOutputData ? 0 : numTransformsPerSplit; const SizeType outputSplitStrideInner = transposeOutputData ? numTransformsPerSplit : 0; // check for non-transposed output assert((transposeInputData) || (size / 2 + 1 == inputData.dim_inner())); // check for transposed output assert((!transposeInputData) || (size / 2 + 1 == inputData.dim_mid())); // determine number of transforms per plane // create plans within each plane transforms_.reserve(inputData.dim_outer() * numSplitsPerPlane); for (SizeType idxOuter = 0; idxOuter < inputData.dim_outer(); ++idxOuter) { for (SizeType idxSplit = 0; idxSplit < numSplitsPerPlane; ++idxSplit) { const SizeType howmany = idxSplit == numSplitsPerPlane - 1 ? numTransformsPerSplit + numTransformsPerPlane % numSplitsPerPlane : numTransformsPerSplit; transforms_.emplace_back(&(inputData(idxOuter, idxSplit * inputSplitStrideMid, idxSplit * inputSplitStrideInner)), &(outputData(idxOuter, idxSplit * outputSplitStrideMid, idxSplit * outputSplitStrideInner)), size, inputStride, outputStride, inputDist, outputDist, howmany); } } } auto execute() -> void override { SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType i = 0; i < transforms_.size(); ++i) { transforms_[i].execute(); } } private: std::vector> transforms_; }; } // namespace spfft #endif SpFFT-0.9.10/src/fft/transform_real_2d_gpu.hpp000066400000000000000000000152121357247005600210720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_REAL_2D_GPU_HPP #define SPFFT_TRANSFORM_REAL_2D_GPU_HPP #include #include #include #include #include "fft/transform_interface.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_runtime_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class TransformReal2DGPU : public TransformGPU { public: using ValueType = T; using ComplexType = gpu::fft::ComplexType; TransformReal2DGPU(GPUArrayView3D spaceDomain, GPUArrayView3D::type> freqDomain, GPUStreamHandle stream, std::shared_ptr> workBuffer) : stream_(std::move(stream)), workBuffer_(std::move(workBuffer)), spaceDomainPtr_(spaceDomain.data()), freqDomainPtr_(freqDomain.data()) { assert(disjoint(spaceDomain, freqDomain)); assert(workBuffer_); assert(spaceDomain.dim_outer() == freqDomain.dim_outer()); assert(spaceDomain.dim_mid() == freqDomain.dim_mid()); assert(spaceDomain.dim_inner() / 2 + 1 == freqDomain.dim_inner()); int rank = 2; int n[2] = {spaceDomain.dim_mid(), spaceDomain.dim_inner()}; int nembedReal[2] = {spaceDomain.dim_mid(), spaceDomain.dim_inner()}; int nembedFreq[2] = {freqDomain.dim_mid(), freqDomain.dim_inner()}; int stride = 1; int distReal = spaceDomain.dim_inner() * spaceDomain.dim_mid(); int distFreq = freqDomain.dim_inner() * freqDomain.dim_mid(); int batch = spaceDomain.dim_outer(); std::size_t worksizeForward = 0; std::size_t worksizeBackward = 0; // create plan gpu::fft::check_result(gpu::fft::create(&planForward_)); gpu::fft::check_result(gpu::fft::create(&planBackward_)); gpu::fft::check_result(gpu::fft::set_auto_allocation(planForward_, 0)); gpu::fft::check_result(gpu::fft::set_auto_allocation(planBackward_, 0)); gpu::fft::check_result(gpu::fft::make_plan_many( planForward_, rank, n, nembedReal, stride, distReal, nembedFreq, stride, distFreq, gpu::fft::TransformType::RealToComplex::value, batch, &worksizeForward)); gpu::fft::check_result(gpu::fft::make_plan_many( planBackward_, rank, n, nembedFreq, stride, distFreq, nembedReal, stride, distReal, gpu::fft::TransformType::ComplexToReal::value, batch, &worksizeBackward)); // set stream gpu::fft::check_result(gpu::fft::set_stream(planForward_, stream_.get())); gpu::fft::check_result(gpu::fft::set_stream(planBackward_, stream_.get())); const std::size_t worksize = worksizeForward > worksizeBackward ? worksizeForward : worksizeBackward; // resize work buffer if necessary if (workBuffer_->size() < worksize) { *workBuffer_ = GPUArray(worksize); } } TransformReal2DGPU(const TransformReal2DGPU& transform) = delete; TransformReal2DGPU(TransformReal2DGPU&& transform) noexcept : stream_(std::move(transform.stream_)), planForward_(std::move(transform.planForward_)), planBackward_(std::move(transform.planBackward_)), workBuffer_(std::move(transform.workBuffer_)), spaceDomainPtr_(transform.spaceDomainPtr_), freqDomainPtr_(transform.freqDomainPtr_) { transform.plan_ = 0; } ~TransformReal2DGPU() { if (planForward_) { gpu::fft::destroy(planForward_); planForward_ = 0; } if (planBackward_) { gpu::fft::destroy(planBackward_); planBackward_ = 0; } } auto operator=(const TransformReal2DGPU& transform) -> TransformReal2DGPU& = delete; auto operator=(TransformReal2DGPU&& transform) noexcept -> TransformReal2DGPU& { if (planForward_) { gpu::fft::destroy(planForward_); planForward_ = 0; } if (planBackward_) { gpu::fft::destroy(planBackward_); planBackward_ = 0; } stream_ = std::move(transform.stream_); planForward_ = std::move(transform.planForward_); planBackward_ = std::move(transform.planBackward_); workBuffer_ = std::move(transform.workBuffer_); spaceDomainPtr_ = transform.spaceDomainPtr_; freqDomainPtr_ = transform.freqDomainPtr_; transform.plan_ = 0; return *this; } inline auto device_id() const noexcept -> int { return stream_.device_id(); } auto forward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(planForward_, workBuffer_->data())); gpu::fft::check_result(gpu::fft::execute(planForward_, spaceDomainPtr_, freqDomainPtr_)); } auto backward() -> void override { gpu::fft::check_result(gpu::fft::set_work_area(planBackward_, workBuffer_->data())); gpu::fft::check_result(gpu::fft::execute(planBackward_, freqDomainPtr_, spaceDomainPtr_)); } private: GPUStreamHandle stream_; gpu::fft::HandleType planForward_ = 0; gpu::fft::HandleType planBackward_ = 0; std::shared_ptr> workBuffer_; T* spaceDomainPtr_; typename gpu::fft::ComplexType::type* freqDomainPtr_; }; } // namespace spfft #endif SpFFT-0.9.10/src/gpu_util/000077500000000000000000000000001357247005600151535ustar00rootroot00000000000000SpFFT-0.9.10/src/gpu_util/complex_conversion.cuh000066400000000000000000000045701357247005600215760ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_COMPLEX_CONVERISON_CUH #define SPFFT_GPU_COMPLEX_CONVERISON_CUH #include "gpu_util/gpu_fft_api.hpp" namespace spfft { template struct ConvertComplex { __device__ __host__ inline static T apply(const U& val) { return val; } }; template <> struct ConvertComplex { __device__ __host__ inline static gpu::fft::ComplexFloatType apply( const gpu::fft::ComplexDoubleType& val) { return gpu::fft::ComplexFloatType{(float)val.x, (float)val.y}; } }; template <> struct ConvertComplex { __device__ __host__ inline static gpu::fft::ComplexDoubleType apply( const gpu::fft::ComplexFloatType& val) { return gpu::fft::ComplexDoubleType{(double)val.x, (double)val.y}; } }; } #endif SpFFT-0.9.10/src/gpu_util/gpu_device_guard.hpp000066400000000000000000000051321357247005600211610ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_DEVICE_GUARD_HPP #define SPFFT_GPU_DEVICE_GUARD_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include #include "gpu_util/gpu_runtime_api.hpp" #include "spfft/exceptions.hpp" namespace spfft { class GPUDeviceGuard { public: explicit GPUDeviceGuard(const int deviceId) : targetDeviceId_(deviceId), originalDeviceId_(0) { gpu::check_status(gpu::get_device(&originalDeviceId_)); if (originalDeviceId_ != deviceId) { gpu::check_status(gpu::set_device(deviceId)); } }; GPUDeviceGuard() = delete; GPUDeviceGuard(const GPUDeviceGuard&) = delete; GPUDeviceGuard(GPUDeviceGuard&&) = delete; auto operator=(const GPUDeviceGuard&) -> GPUDeviceGuard& = delete; auto operator=(GPUDeviceGuard &&) -> GPUDeviceGuard& = delete; ~GPUDeviceGuard() { if (targetDeviceId_ != originalDeviceId_) { gpu::set_device(originalDeviceId_); // no check to avoid throw exeception in destructor } } private: int targetDeviceId_ = 0; int originalDeviceId_ = 0; }; } // namespace spfft #endif #endif SpFFT-0.9.10/src/gpu_util/gpu_event_handle.hpp000066400000000000000000000054341357247005600212010ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_EVENT_HANDLE_HPP #define SPFFT_GPU_EVENT_HANDLE_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include #include "gpu_util/gpu_runtime_api.hpp" #include "spfft/exceptions.hpp" namespace spfft { class GPUEventHandle { public: explicit GPUEventHandle(const bool enableTiming) : deviceId_(0) { gpu::check_status(gpu::get_device(&deviceId_)); gpu::EventType event; const auto flag = enableTiming ? gpu::flag::EventDefault : gpu::flag::EventDisableTiming; gpu::check_status(gpu::event_create_with_flags(&event, flag)); event_ = std::shared_ptr(new gpu::EventType(event), [](gpu::EventType* ptr) { gpu::event_destroy(*ptr); delete ptr; }); }; inline auto get() const -> gpu::EventType { return *event_; } inline auto device_id() const noexcept -> int { return deviceId_; } inline auto record(const gpu::StreamType& stream) const -> void { gpu::check_status(gpu::event_record(*event_, stream)); } inline auto stream_wait(const gpu::StreamType& stream) const -> void { gpu::check_status(gpu::stream_wait_event(stream, *event_, 0)); } private: std::shared_ptr event_; int deviceId_ = 0; }; } // namespace spfft #endif #endif SpFFT-0.9.10/src/gpu_util/gpu_fft_api.cpp000066400000000000000000000045331357247005600201470ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/config.h" #include "gpu_util/gpu_fft_api.hpp" // only declare namespace members if GPU support is enabled #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) namespace spfft { namespace gpu { namespace fft { namespace TransformType { constexpr decltype(ComplexToComplex::value) ComplexToComplex::value; constexpr decltype(ComplexToComplex::value) ComplexToComplex::value; constexpr decltype(RealToComplex::value) RealToComplex::value; constexpr decltype(RealToComplex::value) RealToComplex::value; constexpr decltype(ComplexToReal::value) ComplexToReal::value; constexpr decltype(ComplexToReal::value) ComplexToReal::value; } // namespace TransformType } // namespace fft } // namespace gpu } // namespace spfft #endif SpFFT-0.9.10/src/gpu_util/gpu_fft_api.hpp000066400000000000000000000154711357247005600201570ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_FFT_API_HPP #define SPFFT_GPU_FFT_API_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) #include #define GPU_FFT_PREFIX(val) cufft##val #elif defined(SPFFT_ROCM) #include #define GPU_FFT_PREFIX(val) hipfft##val #endif // only declare namespace members if GPU support is enabled #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include #include "spfft/exceptions.hpp" namespace spfft { namespace gpu { namespace fft { // ================================== // Types // ================================== using ResultType = GPU_FFT_PREFIX(Result); using HandleType = GPU_FFT_PREFIX(Handle); using ComplexFloatType = GPU_FFT_PREFIX(Complex); using ComplexDoubleType = GPU_FFT_PREFIX(DoubleComplex); // Complex type selector template struct ComplexType; template <> struct ComplexType { using type = ComplexDoubleType; }; template <> struct ComplexType { using type = ComplexFloatType; }; // ================================== // Transform types // ================================== namespace TransformDirection { #ifdef SPFFT_CUDA constexpr auto Forward = CUFFT_FORWARD; constexpr auto Backward = CUFFT_INVERSE; #else constexpr auto Forward = HIPFFT_FORWARD; constexpr auto Backward = HIPFFT_BACKWARD; #endif } // namespace TransformDirection // ================================== // Transform types // ================================== namespace TransformType { #ifdef SPFFT_CUDA constexpr auto R2C = CUFFT_R2C; constexpr auto C2R = CUFFT_C2R; constexpr auto C2C = CUFFT_C2C; constexpr auto D2Z = CUFFT_D2Z; constexpr auto Z2D = CUFFT_Z2D; constexpr auto Z2Z = CUFFT_Z2Z; #else constexpr auto R2C = HIPFFT_R2C; constexpr auto C2R = HIPFFT_C2R; constexpr auto C2C = HIPFFT_C2C; constexpr auto D2Z = HIPFFT_D2Z; constexpr auto Z2D = HIPFFT_Z2D; constexpr auto Z2Z = HIPFFT_Z2Z; #endif // Transform type selector template struct ComplexToComplex; template <> struct ComplexToComplex { constexpr static auto value = Z2Z; }; template <> struct ComplexToComplex { constexpr static auto value = C2C; }; // Transform type selector template struct RealToComplex; template <> struct RealToComplex { constexpr static auto value = D2Z; }; template <> struct RealToComplex { constexpr static auto value = R2C; }; // Transform type selector template struct ComplexToReal; template <> struct ComplexToReal { constexpr static auto value = Z2D; }; template <> struct ComplexToReal { constexpr static auto value = C2R; }; } // namespace TransformType // ================================== // Result values // ================================== namespace result { #ifdef SPFFT_CUDA constexpr auto Success = CUFFT_SUCCESS; #else constexpr auto Success = HIPFFT_SUCCESS; #endif } // namespace result // ================================== // Error check functions // ================================== inline auto check_result(ResultType error) -> void { if (error != result::Success) { throw GPUFFTError(); } } // ================================== // Execution function overload // ================================== inline auto execute(HandleType& plan, ComplexDoubleType* iData, double* oData) -> ResultType { return GPU_FFT_PREFIX(ExecZ2D)(plan, iData, oData); } inline auto execute(HandleType& plan, ComplexFloatType* iData, float* oData) -> ResultType { return GPU_FFT_PREFIX(ExecC2R)(plan, iData, oData); } inline auto execute(HandleType& plan, double* iData, ComplexDoubleType* oData) -> ResultType { return GPU_FFT_PREFIX(ExecD2Z)(plan, iData, oData); } inline auto execute(HandleType& plan, float* iData, ComplexFloatType* oData) -> ResultType { return GPU_FFT_PREFIX(ExecR2C)(plan, iData, oData); } inline auto execute(HandleType& plan, ComplexDoubleType* iData, ComplexDoubleType* oData, int direction) -> ResultType { return GPU_FFT_PREFIX(ExecZ2Z)(plan, iData, oData, direction); } inline auto execute(HandleType& plan, ComplexFloatType* iData, ComplexFloatType* oData, int direction) -> ResultType { return GPU_FFT_PREFIX(ExecC2C)(plan, iData, oData, direction); } // ================================== // Forwarding functions of to GPU API // ================================== template inline auto create(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(Create)(std::forward(args)...); } template inline auto make_plan_many(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(MakePlanMany)(std::forward(args)...); } template inline auto set_work_area(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(SetWorkArea)(std::forward(args)...); } template inline auto destroy(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(Destroy)(std::forward(args)...); } template inline auto set_stream(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(SetStream)(std::forward(args)...); } template inline auto set_auto_allocation(ARGS&&... args) -> ResultType { return GPU_FFT_PREFIX(SetAutoAllocation)(std::forward(args)...); } } // namespace fft } // namespace gpu } // namespace spfft #undef GPU_FFT_PREFIX #endif // defined SPFFT_CUDA || SPFFT_ROCM #endif SpFFT-0.9.10/src/gpu_util/gpu_kernel_parameter.hpp000066400000000000000000000036311357247005600220620ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_KERNEL_PARAMETER_HPP #define SPFFT_GPU_KERNEL_PARAMETER_HPP #include "spfft/config.h" namespace spfft { namespace gpu { constexpr int BlockSizeSmall = 128; constexpr int BlockSizeMedium = 256; constexpr int BlockSizeLarge = 512; constexpr int GridSizeSmall = 2160; constexpr int GridSizeMedium = 4320; constexpr int GridSizeLarge = 8640; } } #endif SpFFT-0.9.10/src/gpu_util/gpu_pointer_translation.hpp000066400000000000000000000063071357247005600226430ustar00rootroot00000000000000 /* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_POINTER_TRANSLATION_HPP #define SPFFT_GPU_POINTER_TRANSLATION_HPP #include #include #include "spfft/config.h" namespace spfft { template auto translate_gpu_pointer(const T* inputPointer) -> std::pair { gpu::PointerAttributes attr; auto status = gpu::pointer_get_attributes(&attr, static_cast(inputPointer)); gpu::get_last_error(); // if pointer is not registered, error will be stored and has to be // cleared #ifdef SPFFT_ROCM if (status != gpu::status::Success) { #else if (status == gpu::status::ErrorInvalidValue) { #endif // not registered with cuda -> host pointer const T* devicePtr = nullptr; return {inputPointer, devicePtr}; } else { gpu::check_status(status); return {static_cast(attr.hostPointer), static_cast(attr.devicePointer)}; } } template auto translate_gpu_pointer(T* inputPointer) -> std::pair { gpu::PointerAttributes attr; auto status = gpu::pointer_get_attributes(&attr, static_cast(inputPointer)); gpu::get_last_error(); // if pointer is not registered, error will be stored and has to be // cleared gpu::get_last_error(); #ifdef SPFFT_ROCM if (status != gpu::status::Success) { #else if (status == gpu::status::ErrorInvalidValue) { #endif // not registered with cuda -> host pointer T* devicePtr = nullptr; return {inputPointer, devicePtr}; } else { gpu::check_status(status); return {static_cast(attr.hostPointer), static_cast(attr.devicePointer)}; } } } // namespace spfft #endif SpFFT-0.9.10/src/gpu_util/gpu_runtime.hpp000066400000000000000000000057131357247005600202300ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_RUNTIME_HPP #define SPFFT_GPU_RUNTIME_HPP #include "gpu_util/gpu_runtime_api.hpp" #include "spfft/config.h" #ifdef SPFFT_ROCM #include #endif namespace spfft { #ifdef SPFFT_CUDA template inline auto launch_kernel(F func, const dim3 threadGrid, const dim3 threadBlock, const size_t sharedMemoryBytes, const gpu::StreamType stream, ARGS&&... args) -> void { #ifndef NDEBUG gpu::device_synchronize(); gpu::check_status(gpu::get_last_error()); // before #endif func<<>>(std::forward(args)...); #ifndef NDEBUG gpu::device_synchronize(); gpu::check_status(gpu::get_last_error()); // after #endif } #endif #ifdef SPFFT_ROCM template inline auto launch_kernel(F func, const dim3 threadGrid, const dim3 threadBlock, const size_t sharedMemoryBytes, const gpu::StreamType stream, ARGS&&... args) -> void { #ifndef NDEBUG gpu::device_synchronize(); gpu::check_status(gpu::get_last_error()); // before #endif hipLaunchKernelGGL(func, threadGrid, threadBlock, sharedMemoryBytes, stream, std::forward(args)...); #ifndef NDEBUG gpu::device_synchronize(); gpu::check_status(gpu::get_last_error()); // after #endif } #endif } // namespace spfft #endif SpFFT-0.9.10/src/gpu_util/gpu_runtime_api.hpp000066400000000000000000000210061357247005600210520ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_RUNTIME_RUNTIME_HPP #define SPFFT_GPU_RUNTIME_RUNTIME_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) #include #define GPU_PREFIX(val) cuda##val #elif defined(SPFFT_ROCM) #include #define GPU_PREFIX(val) hip##val #endif // only declare namespace members if GPU support is enabled #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include #include "spfft/exceptions.hpp" namespace spfft { namespace gpu { using StatusType = GPU_PREFIX(Error_t); using StreamType = GPU_PREFIX(Stream_t); using EventType = GPU_PREFIX(Event_t); #ifdef SPFFT_CUDA using PointerAttributes = GPU_PREFIX(PointerAttributes); #else using PointerAttributes = GPU_PREFIX(PointerAttribute_t); #endif namespace status { // error / return values constexpr StatusType Success = GPU_PREFIX(Success); constexpr StatusType ErrorMemoryAllocation = GPU_PREFIX(ErrorMemoryAllocation); constexpr StatusType ErrorLaunchOutOfResources = GPU_PREFIX(ErrorLaunchOutOfResources); constexpr StatusType ErrorInvalidValue = GPU_PREFIX(ErrorInvalidValue); constexpr StatusType ErrorInvalidResourceHandle = GPU_PREFIX(ErrorInvalidResourceHandle); constexpr StatusType ErrorInvalidDevice = GPU_PREFIX(ErrorInvalidDevice); constexpr StatusType ErrorInvalidMemcpyDirection = GPU_PREFIX(ErrorInvalidMemcpyDirection); constexpr StatusType ErrorInvalidDevicePointer = GPU_PREFIX(ErrorInvalidDevicePointer); constexpr StatusType ErrorInitializationError = GPU_PREFIX(ErrorInitializationError); constexpr StatusType ErrorNoDevice = GPU_PREFIX(ErrorNoDevice); constexpr StatusType ErrorNotReady = GPU_PREFIX(ErrorNotReady); constexpr StatusType ErrorUnknown = GPU_PREFIX(ErrorUnknown); constexpr StatusType ErrorPeerAccessNotEnabled = GPU_PREFIX(ErrorPeerAccessNotEnabled); constexpr StatusType ErrorPeerAccessAlreadyEnabled = GPU_PREFIX(ErrorPeerAccessAlreadyEnabled); constexpr StatusType ErrorHostMemoryAlreadyRegistered = GPU_PREFIX(ErrorHostMemoryAlreadyRegistered); constexpr StatusType ErrorHostMemoryNotRegistered = GPU_PREFIX(ErrorHostMemoryNotRegistered); constexpr StatusType ErrorUnsupportedLimit = GPU_PREFIX(ErrorUnsupportedLimit); } // namespace status // flags to pass to GPU API namespace flag { constexpr auto HostRegisterDefault = GPU_PREFIX(HostRegisterDefault); constexpr auto HostRegisterPortable = GPU_PREFIX(HostRegisterPortable); constexpr auto HostRegisterMapped = GPU_PREFIX(HostRegisterMapped); constexpr auto HostRegisterIoMemory = GPU_PREFIX(HostRegisterIoMemory); constexpr auto StreamDefault = GPU_PREFIX(StreamDefault); constexpr auto StreamNonBlocking = GPU_PREFIX(StreamNonBlocking); constexpr auto MemoryTypeHost = GPU_PREFIX(MemoryTypeHost); constexpr auto MemoryTypeDevice = GPU_PREFIX(MemoryTypeDevice); #if (CUDART_VERSION >= 10000) constexpr auto MemoryTypeUnregistered = GPU_PREFIX(MemoryTypeUnregistered); constexpr auto MemoryTypeManaged = GPU_PREFIX(MemoryTypeManaged); #endif constexpr auto MemcpyHostToDevice = GPU_PREFIX(MemcpyHostToDevice); constexpr auto MemcpyDeviceToHost = GPU_PREFIX(MemcpyDeviceToHost); constexpr auto EventDefault = GPU_PREFIX(EventDefault); constexpr auto EventBlockingSync = GPU_PREFIX(EventBlockingSync); constexpr auto EventDisableTiming = GPU_PREFIX(EventDisableTiming); constexpr auto EventInterprocess = GPU_PREFIX(EventInterprocess); } // namespace flag // ================================== // Error check function // ================================== inline auto check_status(StatusType error) -> void { if (error != status::Success) { if (error == status::ErrorMemoryAllocation) throw GPUAllocationError(); if (error == status::ErrorLaunchOutOfResources) throw GPULaunchError(); if (error == status::ErrorNoDevice) throw GPUNoDeviceError(); if (error == status::ErrorInvalidValue) throw GPUInvalidValueError(); if (error == status::ErrorInvalidDevicePointer) throw GPUInvalidDevicePointerError(); throw GPUError(); } } // ================================== // Forwarding functions to GPU API // ================================== template inline auto host_register(ARGS&&... args) -> StatusType { return GPU_PREFIX(HostRegister)(std::forward(args)...); } template inline auto host_unregister(ARGS&&... args) -> StatusType { return GPU_PREFIX(HostUnregister)(std::forward(args)...); } template inline auto stream_create_with_flags(ARGS&&... args) -> StatusType { return GPU_PREFIX(StreamCreateWithFlags)(std::forward(args)...); } template inline auto stream_destroy(ARGS&&... args) -> StatusType { return GPU_PREFIX(StreamDestroy)(std::forward(args)...); } template inline auto stream_wait_event(ARGS&&... args) -> StatusType { return GPU_PREFIX(StreamWaitEvent)(std::forward(args)...); } template inline auto event_create_with_flags(ARGS&&... args) -> StatusType { return GPU_PREFIX(EventCreateWithFlags)(std::forward(args)...); } template inline auto event_destroy(ARGS&&... args) -> StatusType { return GPU_PREFIX(EventDestroy)(std::forward(args)...); } template inline auto event_record(ARGS&&... args) -> StatusType { return GPU_PREFIX(EventRecord)(std::forward(args)...); } template inline auto malloc(ARGS&&... args) -> StatusType { return GPU_PREFIX(Malloc)(std::forward(args)...); } template inline auto free(ARGS&&... args) -> StatusType { return GPU_PREFIX(Free)(std::forward(args)...); } template inline auto memcpy(ARGS&&... args) -> StatusType { return GPU_PREFIX(Memcpy)(std::forward(args)...); } template inline auto memcpy_async(ARGS&&... args) -> StatusType { return GPU_PREFIX(MemcpyAsync)(std::forward(args)...); } template inline auto get_device(ARGS&&... args) -> StatusType { return GPU_PREFIX(GetDevice)(std::forward(args)...); } template inline auto set_device(ARGS&&... args) -> StatusType { return GPU_PREFIX(SetDevice)(std::forward(args)...); } template inline auto get_device_count(ARGS&&... args) -> StatusType { return GPU_PREFIX(GetDeviceCount)(std::forward(args)...); } template inline auto stream_synchronize(ARGS&&... args) -> StatusType { return GPU_PREFIX(StreamSynchronize)(std::forward(args)...); } template inline auto memset_async(ARGS&&... args) -> StatusType { return GPU_PREFIX(MemsetAsync)(std::forward(args)...); } template inline auto pointer_get_attributes(ARGS&&... args) -> StatusType { return GPU_PREFIX(PointerGetAttributes)(std::forward(args)...); } inline auto get_last_error() -> StatusType { return GPU_PREFIX(GetLastError)(); } inline auto device_synchronize() -> StatusType { return GPU_PREFIX(DeviceSynchronize)(); } } // namespace gpu } // namespace spfft #undef GPU_PREFIX #endif // defined SPFFT_CUDA || SPFFT_ROCM #endif SpFFT-0.9.10/src/gpu_util/gpu_stream_handle.hpp000066400000000000000000000054001357247005600213440ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_STREAM_HANDLE_HPP #define SPFFT_GPU_STREAM_HANDLE_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include #include "gpu_util/gpu_runtime_api.hpp" #include "spfft/exceptions.hpp" namespace spfft { class GPUStreamHandle { public: GPUStreamHandle() : stream_(new gpu::StreamType(0)), deviceId_(0) { gpu::check_status(gpu::get_device(&deviceId_)); } explicit GPUStreamHandle(const bool blockedByDefaultStream) : deviceId_(0) { gpu::check_status(gpu::get_device(&deviceId_)); gpu::StreamType rawStream; if (blockedByDefaultStream) gpu::check_status(gpu::stream_create_with_flags(&rawStream, gpu::flag::StreamDefault)); else gpu::check_status(gpu::stream_create_with_flags(&rawStream, gpu::flag::StreamNonBlocking)); stream_ = std::shared_ptr(new gpu::StreamType(rawStream), [](gpu::StreamType* ptr) { gpu::stream_destroy(*ptr); delete ptr; }); }; inline auto get() const -> gpu::StreamType { return *stream_; } inline auto device_id() const noexcept -> int { return deviceId_; } private: std::shared_ptr stream_; int deviceId_ = 0; }; } // namespace spfft #endif #endif SpFFT-0.9.10/src/gpu_util/gpu_transfer.hpp000066400000000000000000000122101357247005600203570ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_TRANSFER_HPP #define SPFFT_GPU_TRANSFER_HPP #include #include "gpu_util/gpu_stream_handle.hpp" #include "memory/memory_type_trait.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template auto copy_to_gpu(const T& hostArray, U&& gpuArray) -> void { using UType = typename std::remove_reference::type; static_assert(!IsDeviceMemory::value, "First argument must represent host memory!"); static_assert(IsDeviceMemory::value, "Second argument must represent device memory!"); static_assert(sizeof(decltype(*(gpuArray.data()))) == sizeof(decltype(*(hostArray.data()))), "Size of value types must match!"); assert(hostArray.size() == static_cast(gpuArray.size())); gpu::check_status(gpu::memcpy( static_cast(gpuArray.data()), static_cast(hostArray.data()), gpuArray.size() * sizeof(decltype(*(gpuArray.data()))), gpu::flag::MemcpyHostToDevice)); } template auto copy_to_gpu_async(const GPUStreamHandle& stream, const T& hostArray, U&& gpuArray) -> void { using UType = typename std::remove_reference::type; static_assert(!IsDeviceMemory::value, "First argument must represent host memory!"); static_assert(IsDeviceMemory::value, "Second argument must represent device memory!"); static_assert(sizeof(decltype(*(gpuArray.data()))) == sizeof(decltype(*(hostArray.data()))), "Size of value types must match!"); assert(hostArray.size() == static_cast(gpuArray.size())); gpu::check_status(gpu::memcpy_async(static_cast(gpuArray.data()), static_cast(hostArray.data()), gpuArray.size() * sizeof(decltype(*(gpuArray.data()))), gpu::flag::MemcpyHostToDevice, stream.get())); } template auto copy_from_gpu(const T& gpuArray, U&& hostArray) -> void { using UType = typename std::remove_reference::type; static_assert(IsDeviceMemory::value, "First argument must represent device memory!"); static_assert(!IsDeviceMemory::value, "Second argument must represent host memory!"); static_assert(sizeof(decltype(*(gpuArray.data()))) == sizeof(decltype(*(hostArray.data()))), "Size of value types must match!"); assert(hostArray.size() == static_cast(gpuArray.size())); gpu::check_status(gpu::memcpy( static_cast(hostArray.data()), static_cast(gpuArray.data()), hostArray.size() * sizeof(decltype(*(gpuArray.data()))), gpu::flag::MemcpyDeviceToHost)); } template auto copy_from_gpu_async(const GPUStreamHandle& stream, const T& gpuArray, U&& hostArray) -> void { using UType = typename std::remove_reference::type; static_assert(IsDeviceMemory::value, "First argument must represent device memory!"); static_assert(!IsDeviceMemory::value, "Second argument must represent host memory!"); static_assert(sizeof(decltype(*(gpuArray.data()))) == sizeof(decltype(*(hostArray.data()))), "Size of value types must match!"); assert(hostArray.size() == static_cast(gpuArray.size())); gpu::check_status(gpu::memcpy_async(static_cast(hostArray.data()), static_cast(gpuArray.data()), hostArray.size() * sizeof(decltype(*(gpuArray.data()))), gpu::flag::MemcpyDeviceToHost, stream.get())); } } // namespace spfft #endif SpFFT-0.9.10/src/memory/000077500000000000000000000000001357247005600146335ustar00rootroot00000000000000SpFFT-0.9.10/src/memory/aligned_allocation.cpp000066400000000000000000000047051357247005600211550ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "memory/aligned_allocation.hpp" #include #include namespace spfft { namespace memory { auto allocate_aligned(SizeType numBytes, SizeType alignment) -> void* { // check if sizeof(void*) is power of 2 static_assert((sizeof(void*) & (sizeof(void*) - 1)) == 0, "size of void* must by power of 2 for alignment!"); // check if alignment is power of 2 and multiple of sizeof(void*) if (alignment % sizeof(void*) != 0 || ((alignment & (alignment - 1)) != 0)) throw HostAllocationError(); void* ptr; if (posix_memalign(&ptr, alignment, numBytes) != 0) throw HostAllocationError(); return ptr; } auto allocate_aligned(SizeType numBytes) -> void* { static auto pageSize = sysconf(_SC_PAGESIZE); return allocate_aligned(numBytes, static_cast(pageSize)); } auto free_aligned(void* ptr) noexcept -> void { free(ptr); } } // namespace memory } // namespace spfft SpFFT-0.9.10/src/memory/aligned_allocation.hpp000066400000000000000000000101011357247005600211450ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_ALIGNED_ALLOCATOR_HPP #define SPFFT_ALIGNED_ALLOCATOR_HPP #include #include #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" namespace spfft { namespace memory { // Allocate given number of bytes at adress with given alignment. // The alignment must be a multiple of sizeof(void*) and a power of 2 // Throws upon failure. auto allocate_aligned(SizeType numBytes, SizeType alignment) -> void*; // Allocate memory aligned at page boundaries auto allocate_aligned(SizeType numBytes) -> void*; // Free memory allocated with allocate_aligned() function auto free_aligned(void* ptr) noexcept -> void; // construct numElements elements of type T with arguments args at location pointed to by ptr template auto construct_elements_in_place(T* ptr, SizeType numElements, ARGS&&... args) -> void; // deconstruct elements of trivially destructable type in array template ::value, int>::type = 0> auto deconstruct_elements(T* ptr, SizeType numElements) noexcept -> void; // deconstruct elements of non-trivially destructable type in array template ::value, int>::type = 0> auto deconstruct_elements(T* ptr, SizeType numElements) noexcept(std::is_nothrow_destructible::value) -> void; // ====================== // Implementation // ====================== template auto construct_elements_in_place(T* ptr, SizeType numElements, ARGS&&... args) -> void { SizeType constructIdx = 0; try { // construct all elements for (; constructIdx < numElements; ++constructIdx) { new (ptr + constructIdx) T(std::forward(args)...); } } catch (...) { // destruct all elements which did not throw in case of error deconstruct_elements(ptr, constructIdx); throw; } } template ::value, int>::type> auto deconstruct_elements(T*, SizeType) noexcept -> void {} template ::value, int>::type> auto deconstruct_elements(T* ptr, SizeType numElements) noexcept(std::is_nothrow_destructible::value) -> void { for (SizeType destructIdx = 0; destructIdx < numElements; ++destructIdx) { ptr[destructIdx].~T(); } } } // namespace memory } // namespace spfft #endif SpFFT-0.9.10/src/memory/array_view_utility.hpp000066400000000000000000000275631357247005600213140ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_ARRAY_VIEW_UTILITY_HPP #define SPFFT_ARRAY_VIEW_UTILITY_HPP #include #include #include #include #include #include "memory/gpu_array_view.hpp" #include "memory/host_array_view.hpp" #include "memory/memory_type_trait.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template auto disjoint(const T& array1, const U& array2) -> bool { const void* start1 = static_cast(array1.data()); const void* end1 = static_cast(array1.data() + array1.size()); const void* start2 = static_cast(array2.data()); const void* end2 = static_cast(array2.data() + array2.size()); return !(start1 >= start2 && start1 < end2) && !(start2 >= start1 && start2 < end1); } namespace gpu_array_utility_internal { inline auto checked_cast_to_int(const SizeType value) -> int { static_assert(std::is_unsigned::value, "Expected unsigend SizeType"); if (value > static_cast(std::numeric_limits::max())) { throw OverflowError(); } return static_cast(value); } } // namespace gpu_array_utility_internal // ---------------------- // Create array view // ---------------------- template ::value, int>::type = 0> auto create_1d_view(T& array, const SizeType startIdx, const SizeType size) -> HostArrayView1D { assert(array.size() >= startIdx + size); return HostArrayView1D(array.data() + startIdx, size, array.pinned()); } template ::value, int>::type = 0> auto create_1d_view(T& array, const SizeType startIdx, const SizeType size) -> GPUArrayView1D { assert(array.size() >= startIdx + size); return GPUArrayView1D( array.data() + startIdx, gpu_array_utility_internal::checked_cast_to_int(size), array.device_id()); } template ::value, int>::type = 0> auto create_2d_view(T& array, const SizeType startIdx, const SizeType dimOuter, const SizeType dimInner) -> HostArrayView2D { assert(array.size() >= startIdx + dimInner * dimOuter); return HostArrayView2D(array.data() + startIdx, dimOuter, dimInner, array.pinned()); } template ::value, int>::type = 0> auto create_2d_view(T& array, const SizeType startIdx, const SizeType dimOuter, const SizeType dimInner) -> GPUArrayView2D { assert(array.size() >= startIdx + dimInner * dimOuter); // check that entire memory can be adressed with int gpu_array_utility_internal::checked_cast_to_int(dimOuter * dimInner); return GPUArrayView2D( array.data() + startIdx, gpu_array_utility_internal::checked_cast_to_int(dimOuter), gpu_array_utility_internal::checked_cast_to_int(dimInner), array.device_id()); } template ::value, int>::type = 0> auto create_3d_view(T& array, const SizeType startIdx, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner) -> HostArrayView3D { assert(array.size() >= startIdx + dimOuter * dimMid * dimInner); return HostArrayView3D(array.data() + startIdx, dimOuter, dimMid, dimInner, array.pinned()); } template ::value, int>::type = 0> auto create_3d_view(T& array, const SizeType startIdx, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner) -> GPUArrayView3D { assert(array.size() >= startIdx + dimOuter * dimMid * dimInner); // check that entire memory can be adressed with int gpu_array_utility_internal::checked_cast_to_int(dimOuter * dimMid * dimInner); return GPUArrayView3D( array.data() + startIdx, gpu_array_utility_internal::checked_cast_to_int(dimOuter), gpu_array_utility_internal::checked_cast_to_int(dimMid), gpu_array_utility_internal::checked_cast_to_int(dimInner), array.device_id()); } // ------------------------------- // Create array view with new type // ------------------------------ template ::value, int>::type = 0> auto create_new_type_1d_view(T& array, const SizeType size) -> HostArrayView1D { assert(array.size() * sizeof(typename T::ValueType) >= size * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); return HostArrayView1D(reinterpret_cast(array.data()), size, array.pinned()); } template ::value, int>::type = 0> auto create_new_type_1d_view(T& array, const SizeType size) -> GPUArrayView1D { assert(array.size() * sizeof(typename T::ValueType) >= size * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); return GPUArrayView1D(reinterpret_cast(array.data()), gpu_array_utility_internal::checked_cast_to_int(size), array.device_id()); } template ::value, int>::type = 0> auto create_new_type_2d_view(T& array, const SizeType dimOuter, const SizeType dimInner) -> HostArrayView2D { assert(array.size() * sizeof(typename T::ValueType) >= dimOuter * dimInner * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); return HostArrayView2D(reinterpret_cast(array.data()), dimOuter, dimInner, array.pinned()); } template ::value, int>::type = 0> auto create_new_type_2d_view(T& array, const SizeType dimOuter, const SizeType dimInner) -> GPUArrayView2D { assert(array.size() * sizeof(typename T::ValueType) >= dimOuter * dimInner * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); // check that entire memory can be adressed with int gpu_array_utility_internal::checked_cast_to_int(dimOuter * dimInner); return GPUArrayView2D( reinterpret_cast(array.data()), gpu_array_utility_internal::checked_cast_to_int(dimOuter), gpu_array_utility_internal::checked_cast_to_int(dimInner), array.device_id()); } template ::value, int>::type = 0> auto create_new_type_3d_view(T& array, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner) -> HostArrayView3D { assert(array.size() * sizeof(typename T::ValueType) >= dimOuter * dimMid * dimInner * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); return HostArrayView3D(reinterpret_cast(array.data()), dimOuter, dimMid, dimInner, array.pinned()); } template ::value, int>::type = 0> auto create_new_type_3d_view(T& array, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner) -> GPUArrayView3D { assert(array.size() * sizeof(typename T::ValueType) >= dimOuter * dimMid * dimInner * sizeof(U)); static_assert(alignof(typename T::ValueType) % alignof(U) == 0, "Alignment of old type must be multiple of new type alignment"); // check that entire memory can be adressed with int gpu_array_utility_internal::checked_cast_to_int(dimOuter * dimMid * dimInner); return GPUArrayView3D( reinterpret_cast(array.data()), gpu_array_utility_internal::checked_cast_to_int(dimOuter), gpu_array_utility_internal::checked_cast_to_int(dimMid), gpu_array_utility_internal::checked_cast_to_int(dimInner), array.device_id()); } // -------------------------------- // convert scalar and complex views // -------------------------------- template auto convert_to_complex_view(HostArrayView1D view) -> HostArrayView1D> { assert(view.size() % 2 == 0); return HostArrayView1D>(reinterpret_cast*>(view.data()), view.size() / 2, view.pinned()); } template auto convert_to_complex_view(HostArrayView2D view) -> HostArrayView2D> { assert(view.dim_inner() % 2 == 0); return HostArrayView2D>(reinterpret_cast*>(view.data()), view.dim_outer(), view.dim_inner() / 2, view.pinned()); } template auto convert_to_complex_view(HostArrayView3D view) -> HostArrayView3D> { assert(view.dim_inner() % 2 == 0); return HostArrayView3D>(reinterpret_cast*>(view.data()), view.dim_outer(), view.dim_mid(), view.dim_inner() / 2, view.pinned()); } template auto convert_from_complex_view(HostArrayView2D> view) -> HostArrayView1D { return HostArrayView1D(reinterpret_cast(view.data()), view.size() * 2, view.pinned()); } template auto convert_from_complex_view(HostArrayView2D> view) -> HostArrayView3D { return HostArrayView2D(reinterpret_cast(view.data()), view.dim_outer(), view.dim_inner() * 2, view.pinned()); } template auto convert_from_complex_view(HostArrayView3D> view) -> HostArrayView3D { return HostArrayView3D(reinterpret_cast(view.data()), view.dim_outer(), view.dim_mid(), view.dim_inner() * 2, view.pinned()); } } // namespace spfft #endif SpFFT-0.9.10/src/memory/gpu_array.hpp000066400000000000000000000071031357247005600173360ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_ARRAY_HPP #define SPFFT_GPU_ARRAY_HPP #include #include "gpu_util/gpu_runtime_api.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class GPUArray { public: using ValueType = T; static constexpr SizeType ORDER = 1; GPUArray() = default; GPUArray(const SizeType size); GPUArray(const GPUArray& array) = delete; GPUArray(GPUArray&& array) noexcept; ~GPUArray(); auto operator=(const GPUArray& array) -> GPUArray& = delete; auto operator=(GPUArray&& array) noexcept -> GPUArray&; inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return size_ == 0; } inline auto size() const noexcept -> SizeType { return size_; } inline auto device_id() const noexcept -> int { return deviceId_; } private: SizeType size_ = 0; ValueType* data_ = nullptr; int deviceId_ = 0; }; // ====================== // Implementation // ====================== template GPUArray::GPUArray(const SizeType size) : size_(size), data_(nullptr), deviceId_(0) { assert(size >= 0); gpu::check_status(gpu::get_device(&deviceId_)); if (size > 0) { gpu::check_status(gpu::malloc(reinterpret_cast(&data_), size * sizeof(ValueType))); } } template GPUArray::~GPUArray() { if (data_) { // don't check error to avoid throwing exception in destructor gpu::free(data_); data_ = nullptr; size_ = 0; } } template GPUArray::GPUArray(GPUArray&& array) noexcept : size_(array.size_), data_(array.data_), deviceId_(array.deviceId_) { array.data_ = nullptr; array.size_ = 0; } template auto GPUArray::operator=(GPUArray&& array) noexcept -> GPUArray& { if (data_) { gpu::free(data_); } data_ = array.data_; size_ = array.size_; deviceId_ = array.deviceId_; array.data_ = nullptr; array.size_ = 0; return *this; } } // namespace spfft #endif SpFFT-0.9.10/src/memory/gpu_array_const_view.hpp000066400000000000000000000221451357247005600216010ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_ARRAY_CONST_VIEW_HPP #define SPFFT_GPU_ARRAY_CONST_VIEW_HPP #include #include #include "memory/gpu_array_view.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #if defined(__CUDACC__) || defined(__HIPCC__) #include "gpu_util/gpu_runtime.hpp" #endif namespace spfft { // T must be build-in type template class GPUArrayConstView1D { public: using ValueType = T; static constexpr SizeType ORDER = 1; GPUArrayConstView1D() = default; GPUArrayConstView1D(const ValueType* data, const int size, const int deviceId); GPUArrayConstView1D(const GPUArrayView1D&); // conversion allowed #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idx) const -> ValueType { assert(idx < size_); #if __CUDA_ARCH__ >= 350 || defined(__HIPCC__) return __ldg(data_ + idx); #else return data_[idx]; #endif } __host__ __device__ inline auto empty() const noexcept -> bool { return size_ == 0; } __host__ __device__ inline auto size() const noexcept -> int { return size_; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto empty() const noexcept -> bool { return size_ == 0; } inline auto size() const noexcept -> int { return size_; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int size_ = 0; const ValueType* data_ = nullptr; int deviceId_ = 0; }; // T must be build-in type template class GPUArrayConstView2D { public: using ValueType = T; static constexpr SizeType ORDER = 2; GPUArrayConstView2D() = default; GPUArrayConstView2D(const ValueType* data, const int dimOuter, const int dimInner, const int deviceId); GPUArrayConstView2D(const GPUArrayView2D&); // conversion allowed #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idxOuter, const int idxInner) const -> ValueType { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); #if __CUDA_ARCH__ >= 350 || defined(__HIPCC__) return __ldg(data_ + (idxOuter * dims_[1]) + idxInner); #else return data_[(idxOuter * dims_[1]) + idxInner]; #endif } __host__ __device__ inline auto index(const int idxOuter, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1]) + idxInner; } __host__ __device__ inline auto empty() const noexcept -> bool { return this->size() == 0; } __host__ __device__ inline auto size() const noexcept -> int { return dims_[0] * dims_[1]; } __host__ __device__ inline auto dim_inner() const noexcept -> int { return dims_[1]; } __host__ __device__ inline auto dim_outer() const noexcept -> int { return dims_[0]; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto index(const int idxOuter, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1]) + idxInner; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> int { return dims_[0] * dims_[1]; } inline auto dim_inner() const noexcept -> int { return dims_[1]; } inline auto dim_outer() const noexcept -> int { return dims_[0]; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int dims_[2]; const ValueType* data_ = nullptr; int deviceId_ = 0; }; // T must be build-in type template class GPUArrayConstView3D { public: using ValueType = T; static constexpr SizeType ORDER = 3; GPUArrayConstView3D() = default; GPUArrayConstView3D(const ValueType* data, const int dimOuter, const int dimMid, const int dimInner, const int deviceId); GPUArrayConstView3D(const GPUArrayView3D&); // conversion allowed #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idxOuter, const int idxMid, const int idxInner) const -> ValueType { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); #if __CUDA_ARCH__ >= 350 || defined(__HIPCC__) return __ldg(data_ + (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner); #else return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; #endif } __host__ __device__ inline auto index(const int idxOuter, const int idxMid, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } __host__ __device__ inline auto empty() const noexcept -> bool { return this->size() == 0; } __host__ __device__ inline auto size() const noexcept -> int { return dims_[0] * dims_[1] * dims_[2]; } __host__ __device__ inline auto dim_inner() const noexcept -> int { return dims_[2]; } __host__ __device__ inline auto dim_mid() const noexcept -> int { return dims_[1]; } __host__ __device__ inline auto dim_outer() const noexcept -> int { return dims_[0]; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto index(const int idxOuter, const int idxMid, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> int { return dims_[0] * dims_[1] * dims_[2]; } inline auto dim_inner() const noexcept -> int { return dims_[2]; } inline auto dim_mid() const noexcept -> int { return dims_[1]; } inline auto dim_outer() const noexcept -> int { return dims_[0]; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int dims_[3]; const ValueType* data_ = nullptr; int deviceId_ = 0; }; // ====================== // Implementation // ====================== template GPUArrayConstView1D::GPUArrayConstView1D(const ValueType* data, const int size, const int deviceId) : size_(size), data_(data), deviceId_(deviceId) { assert(!(size != 0 && data == nullptr)); } template GPUArrayConstView1D::GPUArrayConstView1D(const GPUArrayView1D& view) : size_(view.size()), data_(view.data()), deviceId_(view.device_id()) {} template GPUArrayConstView2D::GPUArrayConstView2D(const ValueType* data, const int dimOuter, const int dimInner, const int deviceId) : dims_{dimOuter, dimInner}, data_(data), deviceId_(deviceId) { assert(!(dimOuter != 0 && dimInner != 0 && data == nullptr)); assert(dimOuter >= 0); assert(dimInner >= 0); } template GPUArrayConstView2D::GPUArrayConstView2D(const GPUArrayView2D& view) : dims_{view.dim_outer(), view.dim_inner()}, data_(view.data()), deviceId_(view.device_id()) {} template GPUArrayConstView3D::GPUArrayConstView3D(const ValueType* data, const int dimOuter, const int dimMid, const int dimInner, const int deviceId) : dims_{dimOuter, dimMid, dimInner}, data_(data), deviceId_(deviceId) { assert(!(dimOuter != 0 && dimMid != 0 && dimInner != 0 && data == nullptr)); assert(dimOuter >= 0); assert(dimMid >= 0); assert(dimInner >= 0); } template GPUArrayConstView3D::GPUArrayConstView3D(const GPUArrayView3D& view) : dims_{view.dim_outer(), view.dim_mid(), view.dim_inner()}, data_(view.data()), deviceId_(view.device_id()) {} } // namespace spfft #endif SpFFT-0.9.10/src/memory/gpu_array_view.hpp000066400000000000000000000222741357247005600203760ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GPU_ARRAY_VIEW_HPP #define SPFFT_GPU_ARRAY_VIEW_HPP #include #include #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" #if defined(__CUDACC__) || defined(__HIPCC__) #include "gpu_util/gpu_runtime.hpp" #endif namespace spfft { template class GPUArrayView1D { public: using ValueType = T; static constexpr SizeType ORDER = 1; GPUArrayView1D() = default; GPUArrayView1D(ValueType* data, const int size, const int deviceId); #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idx) -> ValueType& { assert(idx < size_); return data_[idx]; } __device__ inline auto operator()(const int idx) const -> const ValueType& { assert(idx < size_); return data_[idx]; } __host__ __device__ inline auto data() noexcept -> ValueType* { return data_; } __host__ __device__ inline auto data() const noexcept -> const ValueType* { return data_; } __host__ __device__ inline auto empty() const noexcept -> bool { return size_ == 0; } __host__ __device__ inline auto size() const noexcept -> int { return size_; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return size_ == 0; } inline auto size() const noexcept -> int { return size_; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int size_ = 0; ValueType* data_ = nullptr; int deviceId_ = 0; }; template class GPUArrayView2D { public: using ValueType = T; static constexpr SizeType ORDER = 2; GPUArrayView2D() = default; GPUArrayView2D(ValueType* data, const int dimOuter, const int dimInner, const int deviceId); #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idxOuter, const int idxInner) -> ValueType& { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); return data_[(idxOuter * dims_[1]) + idxInner]; } __device__ inline auto operator()(const int idxOuter, const int idxInner) const -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); return data_[(idxOuter * dims_[1]) + idxInner]; } __host__ __device__ inline auto index(const int idxOuter, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1]) + idxInner; } __host__ __device__ inline auto data() noexcept -> ValueType* { return data_; } __host__ __device__ inline auto data() const noexcept -> const ValueType* { return data_; } __host__ __device__ inline auto empty() const noexcept -> bool { return this->size() == 0; } __host__ __device__ inline auto size() const noexcept -> int { return dims_[0] * dims_[1]; } __host__ __device__ inline auto dim_inner() const noexcept -> int { return dims_[1]; } __host__ __device__ inline auto dim_outer() const noexcept -> int { return dims_[0]; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto index(const int idxOuter, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1]) + idxInner; } inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> int { return dims_[0] * dims_[1]; } inline auto dim_inner() const noexcept -> int { return dims_[1]; } inline auto dim_outer() const noexcept -> int { return dims_[0]; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int dims_[2]; ValueType* data_ = nullptr; int deviceId_ = 0; }; template class GPUArrayView3D { public: using ValueType = T; static constexpr SizeType ORDER = 3; GPUArrayView3D() = default; GPUArrayView3D(ValueType* data, const int dimOuter, const int dimMid, const int dimInner, const int deviceId); #if defined(__CUDACC__) || defined(__HIPCC__) __device__ inline auto operator()(const int idxOuter, const int idxMid, const int idxInner) noexcept -> ValueType& { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; } __device__ inline auto operator()(const int idxOuter, const int idxMid, const int idxInner) const noexcept -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; } __host__ __device__ inline auto index(const int idxOuter, const int idxMid, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } __host__ __device__ inline auto data() noexcept -> ValueType* { return data_; } __host__ __device__ inline auto data() const noexcept -> const ValueType* { return data_; } __host__ __device__ inline auto empty() const noexcept -> bool { return this->size() == 0; } __host__ __device__ inline auto size() const noexcept -> int { return dims_[0] * dims_[1] * dims_[2]; } __host__ __device__ inline auto dim_inner() const noexcept -> int { return dims_[2]; } __host__ __device__ inline auto dim_mid() const noexcept -> int { return dims_[1]; } __host__ __device__ inline auto dim_outer() const noexcept -> int { return dims_[0]; } __host__ __device__ inline auto device_id() const noexcept -> int { return deviceId_; } #else inline auto index(const int idxOuter, const int idxMid, const int idxInner) const noexcept -> int { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> int { return dims_[0] * dims_[1] * dims_[2]; } inline auto dim_inner() const noexcept -> int { return dims_[2]; } inline auto dim_mid() const noexcept -> int { return dims_[1]; } inline auto dim_outer() const noexcept -> int { return dims_[0]; } inline auto device_id() const noexcept -> int { return deviceId_; } #endif private: int dims_[3]; ValueType* data_ = nullptr; int deviceId_ = 0; }; // ====================== // Implementation // ====================== template GPUArrayView1D::GPUArrayView1D(ValueType* data, const int size, const int deviceId) : size_(size), data_(data), deviceId_(deviceId) { assert(!(size != 0 && data == nullptr)); } template GPUArrayView2D::GPUArrayView2D(ValueType* data, const int dimOuter, const int dimInner, const int deviceId) : dims_{dimOuter, dimInner}, data_(data), deviceId_(deviceId) { assert(!(dimOuter != 0 && dimInner != 0 && data == nullptr)); assert(dimOuter >= 0); assert(dimInner >= 0); } template GPUArrayView3D::GPUArrayView3D(ValueType* data, const int dimOuter, const int dimMid, const int dimInner, const int deviceId) : dims_{dimOuter, dimMid, dimInner}, data_(data), deviceId_(deviceId) { assert(!(dimOuter != 0 && dimMid != 0 && dimInner != 0 && data == nullptr)); assert(dimOuter >= 0); assert(dimMid >= 0); assert(dimInner >= 0); } } // namespace spfft #endif SpFFT-0.9.10/src/memory/host_array.hpp000066400000000000000000000157541357247005600175330ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_HOST_ARRAY_HPP #define SPFFT_HOST_ARRAY_HPP #include #include #include #include #include "gpu_util/gpu_runtime_api.hpp" #include "memory/aligned_allocation.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { // Fixed sized array with data aligned to page boundaries // and requirements for pinned memory with ROCm. // The data can be pinned in memory, if GPU support is enabled. // The destructor of type T must not throw. template class HostArray { public: static_assert(std::is_nothrow_destructible::value, "Destructor of ValueType for HostArray must be noexcept."); using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 1; // Construct empty array HostArray() noexcept; // Create array with given size. Additional parameters are passed to the // constructor of each element of type T. // Throws exception upon allocation or element construction failure template HostArray(SizeType size, ARGS&&... args); HostArray(const HostArray& array) = delete; HostArray(HostArray&& array) noexcept; ~HostArray() noexcept(std::is_nothrow_destructible::value); auto operator=(const HostArray& array) -> HostArray& = delete; auto operator=(HostArray&& array) noexcept -> HostArray&; inline auto operator[](const SizeType idx) -> ValueType& { assert(idx < size_); return data_[idx]; } inline auto operator[](const SizeType idx) const -> const ValueType& { assert(idx < size_); return data_[idx]; } inline auto operator()(const SizeType idx) -> ValueType& { assert(idx < size_); return data_[idx]; } inline auto operator()(const SizeType idx) const -> const ValueType& { assert(idx < size_); return data_[idx]; } inline auto size() const noexcept -> SizeType { return size_; } inline auto pinned() const noexcept -> bool { return pinned_; } // Attempt to pin memory. Return true on success and false otherwise auto pin_memory() noexcept -> bool; // Unpin memory if pinned. Does nothing otherwise auto unpin_memory() noexcept -> void; inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto begin() noexcept -> Iterator { return data_; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() noexcept -> Iterator { return data_ + size_; } inline auto end() const noexcept -> ConstIterator { return data_ + size_; } inline auto cend() const noexcept -> ConstIterator { return data_ + size_; } // undefined behaviour for empty array inline auto front() -> ValueType& { return data_[0]; } // undefined behaviour for empty array inline auto front() const -> const ValueType& { return data_[0]; } // undefined behaviour for empty array inline auto back() -> ValueType& { return data_[size_ - 1]; } // undefined behaviour for empty array inline auto back() const -> const ValueType& { return data_[size_ - 1]; } inline auto empty() const noexcept -> bool { return size_ == 0; } private: T* data_ = nullptr; SizeType size_ = 0; bool pinned_ = false; }; // ====================== // Implementation // ====================== template HostArray::HostArray() noexcept : data_(nullptr), size_(0), pinned_(false) {} template template HostArray::HostArray(SizeType size, ARGS&&... args) : data_(static_cast(memory::allocate_aligned(size * sizeof(T)))), size_(size), pinned_(false) { try { memory::construct_elements_in_place(data_, size, std::forward(args)...); } catch (...) { size_ = 0; memory::free_aligned(data_); data_ = nullptr; throw; } } template HostArray::HostArray(HostArray&& array) noexcept : data_(nullptr), size_(0), pinned_(false) { data_ = array.data_; array.data_ = nullptr; size_ = array.size_; array.size_ = 0; pinned_ = array.pinned_; array.pinned_ = false; } template HostArray::~HostArray() noexcept(std::is_nothrow_destructible::value) { if (data_) { this->unpin_memory(); memory::deconstruct_elements(data_, size_); memory::free_aligned(data_); data_ = nullptr; size_ = 0; } assert(data_ == nullptr); assert(size_ == 0); assert(!pinned_); } template auto HostArray::operator=(HostArray&& array) noexcept -> HostArray& { if (data_) { this->unpin_memory(); memory::deconstruct_elements(data_, size_); memory::free_aligned(data_); } data_ = array.data_; array.data_ = nullptr; size_ = array.size_; array.size_ = 0; pinned_ = array.pinned_; array.pinned_ = false; return *this; } template auto HostArray::pin_memory() noexcept -> bool { #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) if (!pinned_ && data_) { if (gpu::host_register(static_cast(data_), size_ * sizeof(ValueType), gpu::flag::HostRegisterDefault) == gpu::status::Success) { pinned_ = true; } } #endif return pinned_; } template auto HostArray::unpin_memory() noexcept -> void { #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) if (pinned_) { gpu::host_unregister((void*)data_); pinned_ = false; } #endif } } // namespace spfft #endif SpFFT-0.9.10/src/memory/host_array_const_view.hpp000066400000000000000000000217241357247005600217650ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_HOST_ARRAY_CONST_VIEW_HPP #define SPFFT_HOST_ARRAY_CONST_VIEW_HPP #include #include #include "memory/host_array_view.hpp" #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class HostArrayConstView1D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 1; HostArrayConstView1D() = default; HostArrayConstView1D(const HostArrayConstView1D&) = default; HostArrayConstView1D(HostArrayConstView1D&&) = default; HostArrayConstView1D(const ValueType* data, const SizeType size, const bool pinned); // conversion from non-const view HostArrayConstView1D(const HostArrayView1D& view) : size_(view.size()), pinned_(view.pinned()), data_(view.data()) {} inline auto operator()(const SizeType idx) const -> const ValueType& { assert(idx < size_); return data_[idx]; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return size_ == 0; } inline auto size() const noexcept -> SizeType { return size_; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() const noexcept -> ConstIterator { return data_ + size_; } inline auto cend() const noexcept -> ConstIterator { return data_ + size_; } private: SizeType size_ = 0; bool pinned_ = false; const ValueType* data_ = nullptr; }; template class HostArrayConstView2D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 2; HostArrayConstView2D() = default; HostArrayConstView2D(const HostArrayConstView2D&) = default; HostArrayConstView2D(HostArrayConstView2D&&) = default; HostArrayConstView2D(const ValueType* data, const SizeType dimOuter, const SizeType dimInner, const bool pinned); HostArrayConstView2D(const ValueType* data, const std::array& dims, const bool pinned); // conversion from non-const view HostArrayConstView2D(const HostArrayView2D& view) : dims_({view.dim_outer(), view.dim_inner()}), pinned_(view.pinned()), data_(view.data()) {} inline auto operator()(const SizeType idxOuter, const SizeType idxInner) const -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); return data_[(idxOuter * dims_[1]) + idxInner]; } inline auto index(const SizeType idxOuter, const SizeType idxInner) const noexcept -> SizeType { return (idxOuter * dims_[1]) + idxInner; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> SizeType { return dims_[0] * dims_[1]; } inline auto dim_inner() const noexcept -> SizeType { return dims_[1]; } inline auto dim_outer() const noexcept -> SizeType { return dims_[0]; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() const noexcept -> ConstIterator { return data_ + size(); } inline auto cend() const noexcept -> ConstIterator { return data_ + size(); } private: std::array dims_ = {0, 0}; bool pinned_ = false; const ValueType* data_ = nullptr; }; template class HostArrayConstView3D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 3; HostArrayConstView3D() = default; HostArrayConstView3D(const HostArrayConstView3D&) = default; HostArrayConstView3D(HostArrayConstView3D&&) = default; HostArrayConstView3D(const ValueType* data, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner, const bool pinned); HostArrayConstView3D(const ValueType* data, const std::array& dims, const bool pinned); // conversion from non-const view HostArrayConstView3D(const HostArrayView3D& view) : dims_({view.dim_outer(), view.dim_mid(), view.dim_inner()}), pinned_(view.pinned()), data_(view.data()) {} inline auto operator()(const SizeType idxOuter, const SizeType idxMid, const SizeType idxInner) const noexcept -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; } inline auto index(const SizeType idxOuter, const SizeType idxMid, const SizeType idxInner) const noexcept -> SizeType { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> SizeType { return dims_[0] * dims_[1] * dims_[2]; } inline auto dim_inner() const noexcept -> SizeType { return dims_[2]; } inline auto dim_mid() const noexcept -> SizeType { return dims_[1]; } inline auto dim_outer() const noexcept -> SizeType { return dims_[0]; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() const noexcept -> ConstIterator { return data_ + size(); } inline auto cend() const noexcept -> ConstIterator { return data_ + size(); } private: std::array dims_ = {0, 0, 0}; bool pinned_ = false; const ValueType* data_ = nullptr; }; // ====================== // Implementation // ====================== template HostArrayConstView1D::HostArrayConstView1D(const ValueType* data, const SizeType size, const bool pinned) : size_(size), pinned_(pinned), data_(data) { assert(!(size != 0 && data == nullptr)); } template HostArrayConstView2D::HostArrayConstView2D(const ValueType* data, const SizeType dimOuter, const SizeType dimInner, const bool pinned) : dims_({dimOuter, dimInner}), pinned_(pinned), data_(data) {} template HostArrayConstView2D::HostArrayConstView2D(const ValueType* data, const std::array& dims, const bool pinned) : dims_(dims), pinned_(pinned), data_(data) {} template HostArrayConstView3D::HostArrayConstView3D(const ValueType* data, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner, const bool pinned) : dims_({dimOuter, dimMid, dimInner}), pinned_(pinned), data_(data) {} template HostArrayConstView3D::HostArrayConstView3D(const ValueType* data, const std::array& dims, const bool pinned) : dims_(dims), pinned_(pinned), data_(data) {} } // namespace spfft #endif SpFFT-0.9.10/src/memory/host_array_view.hpp000066400000000000000000000213551357247005600205570ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_HOST_ARRAY_VIEW_HPP #define SPFFT_HOST_ARRAY_VIEW_HPP #include #include #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { template class HostArrayView1D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 1; HostArrayView1D() = default; HostArrayView1D(ValueType* data, const SizeType size, const bool pinned); inline auto operator()(const SizeType idx) -> ValueType& { assert(idx < size_); return data_[idx]; } inline auto operator()(const SizeType idx) const -> const ValueType& { assert(idx < size_); return data_[idx]; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return size_ == 0; } inline auto size() const noexcept -> SizeType { return size_; } inline auto begin() noexcept -> Iterator { return data_; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() noexcept -> Iterator { return data_ + size_; } inline auto end() const noexcept -> ConstIterator { return data_ + size_; } inline auto cend() const noexcept -> ConstIterator { return data_ + size_; } private: SizeType size_ = 0; bool pinned_ = false; ValueType* data_ = nullptr; }; template class HostArrayView2D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 2; HostArrayView2D() = default; HostArrayView2D(ValueType* data, const SizeType dimOuter, const SizeType dimInner, const bool pinned); HostArrayView2D(ValueType* data, const std::array& dims, const bool pinned); inline auto operator()(const SizeType idxOuter, const SizeType idxInner) -> ValueType& { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); return data_[(idxOuter * dims_[1]) + idxInner]; } inline auto operator()(const SizeType idxOuter, const SizeType idxInner) const -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxInner < dims_[1]); return data_[(idxOuter * dims_[1]) + idxInner]; } inline auto index(const SizeType idxOuter, const SizeType idxInner) const noexcept -> SizeType { return (idxOuter * dims_[1]) + idxInner; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> SizeType { return dims_[0] * dims_[1]; } inline auto dim_inner() const noexcept -> SizeType { return dims_[1]; } inline auto dim_outer() const noexcept -> SizeType { return dims_[0]; } inline auto begin() noexcept -> Iterator { return data_; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() noexcept -> Iterator { return data_ + size(); } inline auto end() const noexcept -> ConstIterator { return data_ + size(); } inline auto cend() const noexcept -> ConstIterator { return data_ + size(); } private: std::array dims_ = {0, 0}; bool pinned_ = false; ValueType* data_ = nullptr; }; template class HostArrayView3D { public: using ValueType = T; using Iterator = T*; using ConstIterator = const T*; static constexpr SizeType ORDER = 3; HostArrayView3D() = default; HostArrayView3D(ValueType* data, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner, const bool pinned); HostArrayView3D(ValueType* data, const std::array& dims, const bool pinned); inline auto operator()(const SizeType idxOuter, const SizeType idxMid, const SizeType idxInner) noexcept -> ValueType& { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; } inline auto operator()(const SizeType idxOuter, const SizeType idxMid, const SizeType idxInner) const noexcept -> const ValueType& { assert(idxOuter < dims_[0]); assert(idxMid < dims_[1]); assert(idxInner < dims_[2]); return data_[(idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner]; } inline auto index(const SizeType idxOuter, const SizeType idxMid, const SizeType idxInner) const noexcept -> SizeType { return (idxOuter * dims_[1] + idxMid) * dims_[2] + idxInner; } inline auto pinned() const noexcept -> bool { return pinned_; } inline auto data() noexcept -> ValueType* { return data_; } inline auto data() const noexcept -> const ValueType* { return data_; } inline auto empty() const noexcept -> bool { return this->size() == 0; } inline auto size() const noexcept -> SizeType { return dims_[0] * dims_[1] * dims_[2]; } inline auto dim_inner() const noexcept -> SizeType { return dims_[2]; } inline auto dim_mid() const noexcept -> SizeType { return dims_[1]; } inline auto dim_outer() const noexcept -> SizeType { return dims_[0]; } inline auto begin() noexcept -> Iterator { return data_; } inline auto begin() const noexcept -> ConstIterator { return data_; } inline auto cbegin() const noexcept -> ConstIterator { return data_; } inline auto end() noexcept -> Iterator { return data_ + size(); } inline auto end() const noexcept -> ConstIterator { return data_ + size(); } inline auto cend() const noexcept -> ConstIterator { return data_ + size(); } private: std::array dims_ = {0, 0, 0}; bool pinned_ = false; ValueType* data_ = nullptr; }; // ====================== // Implementation // ====================== template HostArrayView1D::HostArrayView1D(ValueType* data, const SizeType size, const bool pinned) : size_(size), pinned_(pinned), data_(data) { assert(!(size != 0 && data == nullptr)); } template HostArrayView2D::HostArrayView2D(ValueType* data, const SizeType dimOuter, const SizeType dimInner, const bool pinned) : dims_({dimOuter, dimInner}), pinned_(pinned), data_(data) {} template HostArrayView2D::HostArrayView2D(ValueType* data, const std::array& dims, const bool pinned) : dims_(dims), pinned_(pinned), data_(data) {} template HostArrayView3D::HostArrayView3D(ValueType* data, const SizeType dimOuter, const SizeType dimMid, const SizeType dimInner, const bool pinned) : dims_({dimOuter, dimMid, dimInner}), pinned_(pinned), data_(data) {} template HostArrayView3D::HostArrayView3D(ValueType* data, const std::array& dims, const bool pinned) : dims_(dims), pinned_(pinned), data_(data) {} } // namespace spfft #endif SpFFT-0.9.10/src/memory/memory_type_trait.hpp000066400000000000000000000053311357247005600211220ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MEMORY_TYPE_TRAIT_HPP #define SPFFT_MEMORY_TYPE_TRAIT_HPP #include "spfft/config.h" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "memory/gpu_array.hpp" #include "memory/gpu_array_const_view.hpp" #include "memory/gpu_array_view.hpp" #endif namespace spfft { template struct IsDeviceMemory { constexpr static bool value = false; }; #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; template struct IsDeviceMemory> { constexpr static bool value = true; }; #endif } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/000077500000000000000000000000001357247005600151455ustar00rootroot00000000000000SpFFT-0.9.10/src/mpi_util/mpi_check_status.hpp000066400000000000000000000035151357247005600212070ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_CHECK_STATUS_HPP #define SPFFT_MPI_CHECK_STATUS_HPP #include #include "spfft/config.h" #include "spfft/exceptions.hpp" namespace spfft { inline auto mpi_check_status(int status) -> void { if (status != MPI_SUCCESS) { throw MPIError(); } } } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/mpi_communicator_handle.hpp000066400000000000000000000061071357247005600225420ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_COMMUNICATOR_HANDLE_HPP #define SPFFT_MPI_COMMUNICATOR_HANDLE_HPP #include #include #include #include "mpi_util/mpi_check_status.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "util/common_types.hpp" namespace spfft { // MPI Communicator, which creates a duplicate at construction time. // Copies of the object share the same communicator, which is reference counted. class MPICommunicatorHandle { public: MPICommunicatorHandle() : comm_(new MPI_Comm(MPI_COMM_SELF)), size_(1), rank_(0) {} MPICommunicatorHandle(const MPI_Comm& comm) { // create copy of communicator MPI_Comm newComm; mpi_check_status(MPI_Comm_dup(comm, &newComm)); comm_ = std::shared_ptr(new MPI_Comm(newComm), [](MPI_Comm* ptr) { int finalized = 0; MPI_Finalized(&finalized); if (!finalized) { MPI_Comm_free(ptr); } delete ptr; }); int sizeInt, rankInt; mpi_check_status(MPI_Comm_size(*comm_, &sizeInt)); mpi_check_status(MPI_Comm_rank(*comm_, &rankInt)); if (sizeInt < 1 || rankInt < 0) { throw MPIError(); } rank_ = static_cast(rankInt); size_ = static_cast(sizeInt); } inline auto get() const -> const MPI_Comm& { return *comm_; } inline auto size() const noexcept -> SizeType { return size_; } inline auto rank() const noexcept -> SizeType { return rank_; } private: std::shared_ptr comm_ = nullptr; SizeType size_ = 1; SizeType rank_ = 0; }; } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/mpi_datatype_handle.hpp000066400000000000000000000117131357247005600216540ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_DATATYPE_HANDLE_HPP #define SPFFT_MPI_DATATYPE_HANDLE_HPP #include #include #include #include "mpi_util/mpi_check_status.hpp" #include "spfft/config.h" namespace spfft { // Storage for MPI datatypes class MPIDatatypeHandle { public: MPIDatatypeHandle() = default; // Create custom datatype with ownership // Does not call MPI_Type_commit! // Can take predifined MPI types such as MPI_DOUBLE, on which MPI_Type_free() will not be called // NOTE: Freeing a MPI_Datatype on which this type depends on does not affect this type (see "The // MPI core") MPIDatatypeHandle(const MPI_Datatype& mpiType) { assert(mpiType != MPI_DATATYPE_NULL); int numIntegers, numAddresses, numDatatypes, combiner; mpi_check_status( MPI_Type_get_envelope(mpiType, &numIntegers, &numAddresses, &numDatatypes, &combiner)); if (combiner != MPI_COMBINER_NAMED && combiner != MPI_COMBINER_DUP) { // take ownership and call MPI_Type_free upon release type_ = std::shared_ptr(new MPI_Datatype(mpiType), [](MPI_Datatype* ptr) { assert(*ptr != MPI_DATATYPE_NULL); int finalized = 0; MPI_Finalized(&finalized); if (!finalized) { MPI_Type_free(ptr); } delete ptr; }); } else { // only copy type descriptor, will not call MPI_Type_free() type_ = std::make_shared(mpiType); } } inline auto get() const -> const MPI_Datatype& { assert(type_); assert(*type_ != MPI_DATATYPE_NULL); return *type_; } inline auto empty() const noexcept -> bool { return type_ == nullptr; } inline static MPIDatatypeHandle create_contiguous(int count, MPI_Datatype oldType) { MPI_Datatype newType; mpi_check_status(MPI_Type_contiguous(count, oldType, &newType)); mpi_check_status(MPI_Type_commit(&newType)); return MPIDatatypeHandle(newType); } inline static MPIDatatypeHandle create_vector(int count, int blocklength, int stride, MPI_Datatype oldType) { MPI_Datatype newType; mpi_check_status(MPI_Type_vector(count, blocklength, stride, oldType, &newType)); mpi_check_status(MPI_Type_commit(&newType)); return MPIDatatypeHandle(newType); } inline static MPIDatatypeHandle create_hindexed(int count, const int arrayOfBlocklengths[], const MPI_Aint arrayOfDispls[], MPI_Datatype oldType) { MPI_Datatype newType; mpi_check_status( MPI_Type_create_hindexed(count, arrayOfBlocklengths, arrayOfDispls, oldType, &newType)); mpi_check_status(MPI_Type_commit(&newType)); return MPIDatatypeHandle(newType); } inline static MPIDatatypeHandle create_subarray(int ndims, const int arrayOfSizes[], const int arrayOfSubsizes[], const int arrayOfStarts[], int order, MPI_Datatype oldType) { MPI_Datatype newType; mpi_check_status(MPI_Type_create_subarray(ndims, arrayOfSizes, arrayOfSubsizes, arrayOfStarts, order, oldType, &newType)); mpi_check_status(MPI_Type_commit(&newType)); return MPIDatatypeHandle(newType); } private: std::shared_ptr type_ = nullptr; }; } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/mpi_init_handle.hpp000066400000000000000000000051341357247005600210040ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_INIT_HANDLE_HPP #define SPFFT_MPI_INIT_HANDLE_HPP #include #include "mpi_util/mpi_check_status.hpp" #include "spfft/config.h" namespace spfft { // MPI Communicator, which creates a duplicate at construction time. // Copies of the object share the same communicator, which is reference counted. class MPIInitHandle { public: MPIInitHandle(int& argc, char**& argv, bool callFinalize) : callFinalize_(callFinalize) { int initialized; MPI_Initialized(&initialized); if (!initialized) { // MPI_Init(&argc, &argv); int provided; mpi_check_status(MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided)); } } // unmovable MPIInitHandle(const MPIInitHandle& other) = delete; MPIInitHandle(MPIInitHandle&& other) = delete; auto operator=(const MPIInitHandle& other) -> MPIInitHandle& = delete; auto operator=(MPIInitHandle&& other) -> MPIInitHandle& = delete; ~MPIInitHandle() { if (callFinalize_) { MPI_Finalize(); } } private: bool callFinalize_ = false; }; } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/mpi_match_elementary_type.hpp000066400000000000000000000070741357247005600231150ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_MATCH_ELEMENTARY_TYPE_HPP #define SPFFT_MPI_MATCH_ELEMENTARY_TYPE_HPP #include #include "spfft/config.h" namespace spfft { template struct MPIMatchElementaryType; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_CHAR; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_SHORT; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_INT; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_LONG; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_LONG_LONG; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_SIGNED_CHAR; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_UNSIGNED_CHAR; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_UNSIGNED_SHORT; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_UNSIGNED; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_UNSIGNED_LONG; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_UNSIGNED_LONG_LONG; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_FLOAT; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_DOUBLE; } }; template <> struct MPIMatchElementaryType { inline static auto get() -> MPI_Datatype { return MPI_LONG_DOUBLE; } }; } // namespace spfft #endif SpFFT-0.9.10/src/mpi_util/mpi_request_handle.hpp000066400000000000000000000047111357247005600215310ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MPI_REQUEST_HANDLE_HPP #define SPFFT_MPI_REQUEST_HANDLE_HPP #include #include #include #include "mpi_util/mpi_check_status.hpp" #include "spfft/config.h" namespace spfft { // Storage for MPI datatypes class MPIRequestHandle { public: MPIRequestHandle() = default; MPIRequestHandle(const MPIRequestHandle&) = delete; MPIRequestHandle(MPIRequestHandle&&) = default; auto operator=(const MPIRequestHandle& other) -> MPIRequestHandle& = delete; auto operator=(MPIRequestHandle&& other) -> MPIRequestHandle& = default; inline auto get_and_activate() -> MPI_Request* { activated_ = true; return &mpiRequest_; } inline auto wait_if_active() -> void { if (activated_) { activated_ = false; MPI_Wait(&mpiRequest_, MPI_STATUS_IGNORE); } } private: MPI_Request mpiRequest_ = MPI_REQUEST_NULL; bool activated_ = false; }; } // namespace spfft #endif SpFFT-0.9.10/src/parameters/000077500000000000000000000000001357247005600154665ustar00rootroot00000000000000SpFFT-0.9.10/src/parameters/parameters.cpp000066400000000000000000000157421357247005600203460ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "parameters/parameters.hpp" #include #include #include #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #endif namespace spfft { #ifdef SPFFT_MPI Parameters::Parameters(const MPICommunicatorHandle& comm, const SpfftTransformType transformType, const SizeType dimX, const SizeType dimY, const SizeType dimZ, const SizeType numLocalXYPlanes, const SizeType numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) : transformType_(transformType), dimX_(dimX), dimXFreq_(transformType == SPFFT_TRANS_R2C ? dimX / 2 + 1 : dimX), dimY_(dimY), dimZ_(dimZ), totalNumXYPlanes_(dimZ), comm_rank_(comm.rank()), comm_size_(comm.size()) { // helper struct to exchange information struct TransposeParameter { SizeType dimX; SizeType dimY; SizeType dimZ; SizeType numLocalXYPlanes; SizeType numLocalZSticks; SizeType numLocalElements; }; // Only index triplets supported (for now) if (indexFormat != SPFFT_INDEX_TRIPLETS) { throw InternalError(); } // convert indices to internal format std::vector localStickIndices; std::tie(freqValueIndices_, localStickIndices) = convert_index_triplets(transformType == SPFFT_TRANS_R2C, dimX, dimY, dimZ, numLocalElements, indices, indices + 1, indices + 2, 3); stickIndicesPerRank_ = create_distributed_transform_indices(comm, std::move(localStickIndices)); check_stick_duplicates(stickIndicesPerRank_); const SizeType numLocalZSticks = stickIndicesPerRank_[comm.rank()].size(); TransposeParameter paramLocal = TransposeParameter{dimX, dimY, dimZ, numLocalXYPlanes, numLocalZSticks, numLocalElements}; // exchange local parameters MPIDatatypeHandle parameterType = MPIDatatypeHandle::create_contiguous( sizeof(TransposeParameter) / sizeof(SizeType), MPIMatchElementaryType::get()); std::vector paramPerRank(comm.size()); mpi_check_status(MPI_Allgather(¶mLocal, 1, parameterType.get(), paramPerRank.data(), 1, parameterType.get(), comm.get())); // Check parameters SizeType numZSticksTotal = 0; SizeType numXYPlanesTotal = 0; for (const auto& p : paramPerRank) { // dimensions must match for all ranks if (p.dimX != paramLocal.dimX || p.dimY != paramLocal.dimY || p.dimZ != paramLocal.dimZ) { throw MPIParameterMismatchError(); } numZSticksTotal += p.numLocalZSticks; numXYPlanesTotal += p.numLocalXYPlanes; } if (numZSticksTotal > dimX * dimY) { // More z sticks than possible throw MPIParameterMismatchError(); } if (numXYPlanesTotal != dimZ) { throw MPIParameterMismatchError(); } // store all parameters in members numZSticksPerRank_.reserve(comm.size()); numXYPlanesPerRank_.reserve(comm.size()); xyPlaneOffsets_.reserve(comm.size()); SizeType startIndex = 0; SizeType xyPlaneOffset = 0; for (const auto& p : paramPerRank) { numZSticksPerRank_.emplace_back(p.numLocalZSticks); numXYPlanesPerRank_.emplace_back(p.numLocalXYPlanes); xyPlaneOffsets_.emplace_back(xyPlaneOffset); startIndex += p.numLocalZSticks; xyPlaneOffset += p.numLocalXYPlanes; totalNumFrequencyDomainElements_ += p.numLocalElements; } maxNumZSticks_ = *std::max_element(numZSticksPerRank_.begin(), numZSticksPerRank_.end()); maxNumXYPlanes_ = *std::max_element(numXYPlanesPerRank_.begin(), numXYPlanesPerRank_.end()); totalNumZSticks_ = std::accumulate(numZSticksPerRank_.begin(), numZSticksPerRank_.end(), SizeType(0)); // check if this rank holds the x=0, y=0 z-stick, which is treated specially for the real to // complex case zeroZeroStickIndex_ = 0; for (const auto& index : stickIndicesPerRank_[comm.rank()]) { if (index == 0) { break; } ++zeroZeroStickIndex_; } } #endif Parameters::Parameters(const SpfftTransformType transformType, const SizeType dimX, const SizeType dimY, const SizeType dimZ, const SizeType numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) : transformType_(transformType), dimX_(dimX), dimXFreq_(transformType == SPFFT_TRANS_R2C ? dimX / 2 + 1 : dimX), dimY_(dimY), dimZ_(dimZ), maxNumXYPlanes_(dimZ), totalNumXYPlanes_(dimZ), totalNumFrequencyDomainElements_(numLocalElements), comm_rank_(0), comm_size_(1), numXYPlanesPerRank_(1, dimZ), xyPlaneOffsets_(1, 0) { // Only index triplets supported (for now) if (indexFormat != SPFFT_INDEX_TRIPLETS) { throw InternalError(); } std::vector localStickIndices; std::tie(freqValueIndices_, localStickIndices) = convert_index_triplets(transformType == SPFFT_TRANS_R2C, dimX, dimY, dimZ, numLocalElements, indices, indices + 1, indices + 2, 3); stickIndicesPerRank_.emplace_back(std::move(localStickIndices)); check_stick_duplicates(stickIndicesPerRank_); maxNumZSticks_ = stickIndicesPerRank_[0].size(); totalNumZSticks_ = stickIndicesPerRank_[0].size(); numZSticksPerRank_.assign(1, stickIndicesPerRank_[0].size()); zeroZeroStickIndex_ = 0; for (const auto& index : stickIndicesPerRank_[0]) { if (index == 0) { break; } ++zeroZeroStickIndex_; } } } // namespace spfft SpFFT-0.9.10/src/parameters/parameters.hpp000066400000000000000000000142501357247005600203440ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_PARAMETERS_HPP #define SPFFT_PARAMETERS_HPP #include #include #include #include #include "compression/indices.hpp" #include "memory/host_array_const_view.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "spfft/types.h" #include "util/common_types.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #endif namespace spfft { class Parameters { public: #ifdef SPFFT_MPI Parameters(const MPICommunicatorHandle& comm, const SpfftTransformType transformType, const SizeType dimX, const SizeType dimY, const SizeType dimZ, const SizeType numLocalXYPlanes, const SizeType numLocalElements, SpfftIndexFormatType indexFormat, const int* indices); #endif Parameters(const SpfftTransformType transformType, const SizeType dimX, const SizeType dimY, const SizeType dimZ, const SizeType numLocalElements, SpfftIndexFormatType indexFormat, const int* indices); inline auto dim_x() const noexcept -> SizeType { return dimX_; } inline auto dim_x_freq() const noexcept -> SizeType { return dimXFreq_; } inline auto dim_y() const noexcept -> SizeType { return dimY_; } inline auto dim_z() const noexcept -> SizeType { return dimZ_; } inline auto max_num_z_sticks() const noexcept -> SizeType { return maxNumZSticks_; } inline auto max_num_xy_planes() const noexcept -> SizeType { return maxNumXYPlanes_; } inline auto total_num_z_sticks() const noexcept -> SizeType { return totalNumZSticks_; } inline auto total_num_xy_planes() const noexcept -> SizeType { return totalNumXYPlanes_; } inline auto transform_type() const noexcept -> SpfftTransformType { return transformType_; } inline auto zero_zero_stick_index() const noexcept -> SizeType { return zeroZeroStickIndex_; } inline auto num_xy_planes(const SizeType rank) const -> SizeType { assert(rank < numXYPlanesPerRank_.size()); return numXYPlanesPerRank_[rank]; } inline auto local_num_xy_planes() const -> SizeType { assert(comm_rank_ < numXYPlanesPerRank_.size()); return numXYPlanesPerRank_[comm_rank_]; } inline auto xy_plane_offset(const SizeType rank) const -> SizeType { assert(rank < numXYPlanesPerRank_.size()); return xyPlaneOffsets_[rank]; } inline auto local_xy_plane_offset() const -> SizeType { assert(comm_rank_ < numXYPlanesPerRank_.size()); return xyPlaneOffsets_[comm_rank_]; } inline auto num_z_sticks(const SizeType rank) const -> SizeType { assert(rank < numZSticksPerRank_.size()); return numZSticksPerRank_[rank]; } inline auto local_num_z_sticks() const -> SizeType { assert(comm_rank_ < numZSticksPerRank_.size()); return numZSticksPerRank_[comm_rank_]; } inline auto z_stick_xy_indices(const SizeType rank) const -> HostArrayConstView1D { assert(rank < stickIndicesPerRank_.size()); assert(num_z_sticks(rank) == stickIndicesPerRank_[rank].size()); return HostArrayConstView1D(stickIndicesPerRank_[rank].data(), stickIndicesPerRank_[rank].size(), false); } inline auto local_z_stick_xy_indices() const -> HostArrayConstView1D { assert(comm_rank_ < stickIndicesPerRank_.size()); assert(num_z_sticks(comm_rank_) == stickIndicesPerRank_[comm_rank_].size()); return HostArrayConstView1D(stickIndicesPerRank_[comm_rank_].data(), stickIndicesPerRank_[comm_rank_].size(), false); } inline auto local_value_indices() const -> const std::vector& { return freqValueIndices_; } inline auto local_num_elements() const -> SizeType { return freqValueIndices_.size(); } inline auto global_num_elements() const -> SizeType { return totalNumFrequencyDomainElements_; } inline auto global_size() const -> SizeType { return dimX_ * dimY_ * dimZ_; } inline auto comm_rank() const -> SizeType { return comm_rank_; } inline auto comm_size() const -> SizeType { return comm_size_; } private: SpfftTransformType transformType_; SizeType zeroZeroStickIndex_ = std::numeric_limits::max(); SizeType dimX_ = 0; SizeType dimXFreq_ = 0; SizeType dimY_ = 0; SizeType dimZ_ = 0; SizeType maxNumZSticks_ = 0; SizeType maxNumXYPlanes_ = 0; SizeType totalNumZSticks_ = 0; SizeType totalNumXYPlanes_ = 0; SizeType totalNumFrequencyDomainElements_ = 0; SizeType comm_rank_ = 0; SizeType comm_size_ = 1; std::vector numZSticksPerRank_; std::vector numXYPlanesPerRank_; std::vector xyPlaneOffsets_; std::vector> stickIndicesPerRank_; std::vector freqValueIndices_; }; } // namespace spfft #endif SpFFT-0.9.10/src/spfft/000077500000000000000000000000001357247005600144455ustar00rootroot00000000000000SpFFT-0.9.10/src/spfft/grid.cpp000066400000000000000000000240631357247005600161030ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/grid.hpp" #include "spfft/grid.h" #include "spfft/grid_internal.hpp" namespace spfft { Grid::Grid(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads) : grid_(new GridInternal(maxDimX, maxDimY, maxDimZ, maxNumLocalZColumns, processingUnit, maxNumThreads)) {} #ifdef SPFFT_MPI Grid::Grid(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType) : grid_(new GridInternal(maxDimX, maxDimY, maxDimZ, maxNumLocalZColumns, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType)) {} #endif Grid::Grid(const Grid& grid) : grid_(new GridInternal(*(grid.grid_))) {} Grid& Grid::operator=(const Grid& grid) { grid_.reset(new GridInternal(*(grid.grid_))); return *this; } Transform Grid::create_transform(SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) const { return Transform(grid_, processingUnit, transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices); } int Grid::max_dim_x() const { return grid_->max_dim_x(); } int Grid::max_dim_y() const { return grid_->max_dim_y(); } int Grid::max_dim_z() const { return grid_->max_dim_z(); } int Grid::max_num_local_z_columns() const { return grid_->max_num_local_z_columns(); } int Grid::max_local_z_length() const { return grid_->max_num_local_xy_planes(); } SpfftProcessingUnitType Grid::processing_unit() const { return grid_->processing_unit(); } int Grid::device_id() const { return grid_->device_id(); } int Grid::num_threads() const { return grid_->num_threads(); } #ifdef SPFFT_MPI MPI_Comm Grid::communicator() const { return grid_->communicator().get(); } #endif } // namespace spfft //--------------------- // C API //--------------------- extern "C" { SpfftError spfft_grid_create(SpfftGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, SpfftProcessingUnitType processingUnit, int maxNumThreads) { try { *grid = new spfft::Grid(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, processingUnit, maxNumThreads); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_grid_create_distributed(SpfftGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType) { try { *grid = new spfft::Grid(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_grid_create_distributed_fortran( SpfftGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, int commFortran, SpfftExchangeType exchangeType) { try { MPI_Comm comm = MPI_Comm_f2c(commFortran); *grid = new spfft::Grid(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif SpfftError spfft_grid_destroy(SpfftGrid grid) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { delete reinterpret_cast(grid); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } grid = nullptr; return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_max_dim_x(SpfftGrid grid, int* dimX) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimX = reinterpret_cast(grid)->max_dim_x(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_max_dim_y(SpfftGrid grid, int* dimY) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimY = reinterpret_cast(grid)->max_dim_y(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_max_dim_z(SpfftGrid grid, int* dimZ) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimZ = reinterpret_cast(grid)->max_dim_z(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_max_num_local_z_columns(SpfftGrid grid, int* maxNumLocalZColumns) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *maxNumLocalZColumns = reinterpret_cast(grid)->max_num_local_z_columns(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_max_local_z_length(SpfftGrid grid, int* maxLocalZLength) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *maxLocalZLength = reinterpret_cast(grid)->max_local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_processing_unit(SpfftGrid grid, SpfftProcessingUnitType* processingUnit) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *processingUnit = reinterpret_cast(grid)->processing_unit(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_device_id(SpfftGrid grid, int* deviceId) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *deviceId = reinterpret_cast(grid)->device_id(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_grid_num_threads(SpfftGrid grid, int* numThreads) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numThreads = reinterpret_cast(grid)->num_threads(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_grid_communicator(SpfftGrid grid, MPI_Comm* comm) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *comm = reinterpret_cast(grid)->communicator(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_grid_communicator_fortran(SpfftGrid grid, int* commFortran) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *commFortran = MPI_Comm_c2f(reinterpret_cast(grid)->communicator()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif } SpFFT-0.9.10/src/spfft/grid_float.cpp000066400000000000000000000253521357247005600172720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/grid_float.hpp" #include "spfft/grid_float.h" #include "spfft/grid_internal.hpp" #ifdef SPFFT_SINGLE_PRECISION namespace spfft { GridFloat::GridFloat(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, SpfftProcessingUnitType processingUnit, int maxNumThreads) : grid_(new GridInternal(maxDimX, maxDimY, maxDimZ, maxNumLocalZColumns, processingUnit, maxNumThreads)) {} #ifdef SPFFT_MPI GridFloat::GridFloat(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZColumns, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType) : grid_(new GridInternal(maxDimX, maxDimY, maxDimZ, maxNumLocalZColumns, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType)) {} #endif GridFloat::GridFloat(const GridFloat& grid) : grid_(new GridInternal(*(grid.grid_))) {} GridFloat& GridFloat::operator=(const GridFloat& grid) { grid_.reset(new GridInternal(*(grid.grid_))); return *this; } TransformFloat GridFloat::create_transform(SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) const { return TransformFloat(grid_, processingUnit, transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices); } int GridFloat::max_dim_x() const { return grid_->max_dim_x(); } int GridFloat::max_dim_y() const { return grid_->max_dim_y(); } int GridFloat::max_dim_z() const { return grid_->max_dim_z(); } int GridFloat::max_num_local_z_columns() const { return grid_->max_num_local_z_columns(); } int GridFloat::max_local_z_length() const { return grid_->max_num_local_xy_planes(); } SpfftProcessingUnitType GridFloat::processing_unit() const { return grid_->processing_unit(); } int GridFloat::device_id() const { return grid_->device_id(); } int GridFloat::num_threads() const { return grid_->num_threads(); } #ifdef SPFFT_MPI MPI_Comm GridFloat::communicator() const { return grid_->communicator().get(); } #endif } // namespace spfft //--------------------- // C API //--------------------- extern "C" { SpfftError spfft_float_grid_create(SpfftFloatGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, SpfftProcessingUnitType processingUnit, int maxNumThreads) { try { *grid = new spfft::GridFloat(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, processingUnit, maxNumThreads); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_float_grid_create_distributed(SpfftFloatGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, MPI_Comm comm, SpfftExchangeType exchangeType) { try { *grid = new spfft::GridFloat(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_float_grid_create_distributed_fortran( SpfftFloatGrid* grid, int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxLocalZLength, SpfftProcessingUnitType processingUnit, int maxNumThreads, int commFortran, SpfftExchangeType exchangeType) { try { MPI_Comm comm = MPI_Comm_f2c(commFortran); *grid = new spfft::GridFloat(maxDimX, maxDimY, maxDimZ, maxNumLocalZSticks, maxLocalZLength, processingUnit, maxNumThreads, comm, exchangeType); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif SpfftError spfft_float_grid_destroy(SpfftFloatGrid grid) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { delete reinterpret_cast(grid); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } grid = nullptr; return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_max_dim_x(SpfftFloatGrid grid, int* dimX) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimX = reinterpret_cast(grid)->max_dim_x(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_max_dim_y(SpfftFloatGrid grid, int* dimY) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimY = reinterpret_cast(grid)->max_dim_y(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_max_dim_z(SpfftFloatGrid grid, int* dimZ) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimZ = reinterpret_cast(grid)->max_dim_z(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_max_num_local_z_columns(SpfftFloatGrid grid, int* maxNumLocalZColumns) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *maxNumLocalZColumns = reinterpret_cast(grid)->max_num_local_z_columns(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_max_local_z_length(SpfftFloatGrid grid, int* maxLocalZLength) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *maxLocalZLength = reinterpret_cast(grid)->max_local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_processing_unit(SpfftFloatGrid grid, SpfftProcessingUnitType* processingUnit) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *processingUnit = reinterpret_cast(grid)->processing_unit(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_device_id(SpfftFloatGrid grid, int* deviceId) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *deviceId = reinterpret_cast(grid)->device_id(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_grid_num_threads(SpfftFloatGrid grid, int* numThreads) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numThreads = reinterpret_cast(grid)->num_threads(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_float_grid_communicator(SpfftFloatGrid grid, MPI_Comm* comm) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *comm = reinterpret_cast(grid)->communicator(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_float_grid_communicator_fortran(SpfftFloatGrid grid, int* commFortran) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *commFortran = MPI_Comm_c2f(reinterpret_cast(grid)->communicator()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif } #endif SpFFT-0.9.10/src/spfft/grid_internal.cpp000066400000000000000000000242701357247005600177770ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/config.h" #include #include #include "spfft/grid_internal.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #endif #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "gpu_util/gpu_device_guard.hpp" #include "gpu_util/gpu_runtime_api.hpp" #endif namespace spfft { template GridInternal::GridInternal(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, SpfftProcessingUnitType executionUnit, int numThreads) : isLocal_(true), executionUnit_(executionUnit), deviceId_(0), numThreads_(numThreads), maxDimX_(maxDimX), maxDimY_(maxDimY), maxDimZ_(maxDimZ), maxNumLocalZSticks_(maxNumLocalZSticks), maxNumLocalXYPlanes_(maxDimZ) { // input check if (maxDimX <= 0 || maxDimY <= 0 || maxDimZ <= 0 || maxNumLocalZSticks < 0) { throw InvalidParameterError(); } if (!(executionUnit & (SpfftProcessingUnitType::SPFFT_PU_HOST | SpfftProcessingUnitType::SPFFT_PU_GPU))) { throw InvalidParameterError(); } // set number of threads to default omp value if not valid if (numThreads < 1) { numThreads = omp_get_max_threads(); numThreads_ = omp_get_max_threads(); } // allocate memory if (executionUnit & SpfftProcessingUnitType::SPFFT_PU_HOST) { arrayHost1_ = HostArray(static_cast(maxDimX * maxDimY * maxDimZ)); arrayHost2_ = HostArray(static_cast(maxDimX * maxDimY * maxDimZ)); } if (executionUnit & SpfftProcessingUnitType::SPFFT_PU_GPU) { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // store device id gpu::check_status(gpu::get_device(&deviceId_)); if (arrayHost1_.empty()) { // not already created for CPU, which always requires at least as much memory arrayHost1_ = HostArray(static_cast(maxNumLocalZSticks * maxDimZ)); arrayHost2_ = HostArray(static_cast(maxDimX * maxDimY * maxDimZ)); } arrayHost1_.pin_memory(); arrayHost2_.pin_memory(); arrayGPU1_ = GPUArray::type>( static_cast(maxNumLocalZSticks * maxDimZ)); arrayGPU2_ = GPUArray::type>( static_cast(maxDimX * maxDimY * maxDimZ)); // each transform will resize the work buffer as needed fftWorkBuffer_.reset(new GPUArray()); #else throw GPUSupportError(); #endif } } #ifdef SPFFT_MPI template GridInternal::GridInternal(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxNumLocalXYPlanes, SpfftProcessingUnitType executionUnit, int numThreads, MPI_Comm comm, SpfftExchangeType exchangeType) : isLocal_(false), executionUnit_(executionUnit), deviceId_(0), numThreads_(numThreads), maxDimX_(maxDimX), maxDimY_(maxDimY), maxDimZ_(maxDimZ), maxNumLocalZSticks_(maxNumLocalZSticks), maxNumLocalXYPlanes_(maxNumLocalXYPlanes), comm_(comm), exchangeType_(exchangeType) { // input check if (static_cast(maxDimX) * static_cast(maxDimY) * static_cast(maxNumLocalXYPlanes) > std::numeric_limits::max()) { throw OverflowError(); } if (static_cast(maxNumLocalZSticks) * static_cast(maxDimZ) > std::numeric_limits::max()) { throw OverflowError(); } if (maxDimX <= 0 || maxDimY <= 0 || maxDimZ <= 0 || maxNumLocalZSticks < 0) { throw InvalidParameterError(); } if (!(executionUnit & (SpfftProcessingUnitType::SPFFT_PU_HOST | SpfftProcessingUnitType::SPFFT_PU_GPU))) { throw InvalidParameterError(); } if (exchangeType != SpfftExchangeType::SPFFT_EXCH_DEFAULT && exchangeType != SpfftExchangeType::SPFFT_EXCH_BUFFERED && exchangeType != SpfftExchangeType::SPFFT_EXCH_BUFFERED_FLOAT && exchangeType != SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED && exchangeType != SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED_FLOAT && exchangeType != SpfftExchangeType::SPFFT_EXCH_UNBUFFERED) { throw InvalidParameterError(); } // compare parameters between ranks { int errorDetected = 0; int exchangeAll = exchangeType; int executionUnitAll = executionUnit; // Bitwise or will lead to a mismatch on at least one rank if not all values are equal mpi_check_status(MPI_Allreduce(MPI_IN_PLACE, &exchangeAll, 1, MPI_INT, MPI_BOR, comm_.get())); mpi_check_status( MPI_Allreduce(MPI_IN_PLACE, &executionUnitAll, 1, MPI_INT, MPI_BOR, comm_.get())); if (exchangeAll != exchangeType || executionUnitAll != executionUnit) { errorDetected = 1; } // check if any rank has detected an error mpi_check_status(MPI_Allreduce(MPI_IN_PLACE, &errorDetected, 1, MPI_INT, MPI_SUM, comm_.get())); if (errorDetected) { throw MPIParameterMismatchError(); } } // set number of threads to default omp value if not valid if (numThreads < 1) { numThreads = omp_get_max_threads(); numThreads_ = omp_get_max_threads(); } // set default exchange type if (exchangeType == SpfftExchangeType::SPFFT_EXCH_DEFAULT) { exchangeType = SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED; exchangeType_ = SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED; } // mark as local if comm size is 1 if (comm_.size() == 1) isLocal_ = true; int requiredSize = 0; switch (exchangeType) { case SpfftExchangeType::SPFFT_EXCH_BUFFERED: { decltype(maxNumLocalXYPlanes_) globalMaxNumXYPlanes = 0; decltype(maxNumLocalZSticks_) globalMaxNumZSticks = 0; MPI_Allreduce(&maxNumLocalXYPlanes_, &globalMaxNumXYPlanes, 1, MPIMatchElementaryType::get(), MPI_MAX, comm); MPI_Allreduce(&maxNumLocalZSticks_, &globalMaxNumZSticks, 1, MPIMatchElementaryType::get(), MPI_MAX, comm); requiredSize = std::max({globalMaxNumXYPlanes * globalMaxNumZSticks * static_cast(comm_.size() + 1), maxDimX_ * maxDimY_ * maxNumLocalXYPlanes_, maxDimZ_ * maxNumLocalZSticks_}); } break; default: { // AUTO or COMPACT_BUFFERED or UNBUFFERED requiredSize = std::max(maxDimX_ * maxDimY_ * maxNumLocalXYPlanes_, maxDimZ_ * maxNumLocalZSticks_); } break; } // Host arrayHost1_ = HostArray(static_cast(requiredSize)); arrayHost2_ = HostArray(static_cast(requiredSize)); // GPU if (executionUnit & SpfftProcessingUnitType::SPFFT_PU_GPU) { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // store device id gpu::check_status(gpu::get_device(&deviceId_)); arrayHost1_.pin_memory(); arrayHost2_.pin_memory(); arrayGPU1_ = GPUArray::type>( static_cast(requiredSize)); arrayGPU2_ = GPUArray::type>( static_cast(requiredSize)); // each transform will resize the work buffer as needed fftWorkBuffer_.reset(new GPUArray()); #else throw GPUSupportError(); #endif } } #endif template GridInternal::GridInternal(const GridInternal& grid) : isLocal_(grid.isLocal_), executionUnit_(grid.executionUnit_), deviceId_(grid.deviceId_), numThreads_(grid.numThreads_), maxDimX_(grid.maxDimX_), maxDimY_(grid.maxDimY_), maxDimZ_(grid.maxDimZ_), maxNumLocalZSticks_(grid.maxNumLocalZSticks_), maxNumLocalXYPlanes_(grid.maxNumLocalXYPlanes_), arrayHost1_(grid.arrayHost1_.size()), arrayHost2_(grid.arrayHost2_.size()) { #ifdef SPFFT_MPI if (!grid.isLocal_) comm_ = MPICommunicatorHandle(grid.comm_.get()); exchangeType_ = grid.exchangeType_; #endif #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) if (grid.executionUnit_ & SPFFT_PU_GPU) { GPUDeviceGuard(grid.device_id()); if (grid.arrayGPU1_.size() > 0) arrayGPU1_ = GPUArray::type>(grid.arrayGPU1_.size()); if (grid.arrayGPU2_.size() > 0) arrayGPU2_ = GPUArray::type>(grid.arrayGPU2_.size()); if (grid.fftWorkBuffer_) fftWorkBuffer_.reset(new GPUArray(grid.fftWorkBuffer_->size())); } #endif } // instatiate templates for float and double template class GridInternal; #ifdef SPFFT_SINGLE_PRECISION template class GridInternal; #endif } // namespace spfft SpFFT-0.9.10/src/spfft/grid_internal.hpp000066400000000000000000000117311357247005600200020ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GRID_INTERNAL_HPP #define SPFFT_GRID_INTERNAL_HPP #include "spfft/config.h" #include #include #include #include "memory/host_array.hpp" #include "spfft/types.h" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include #include "mpi_util/mpi_communicator_handle.hpp" #endif #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "gpu_util/gpu_fft_api.hpp" #include "memory/gpu_array.hpp" #endif namespace spfft { template class GridInternal { public: static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; GridInternal(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, SpfftProcessingUnitType executionUnit, int numThreads); #ifdef SPFFT_MPI GridInternal(int maxDimX, int maxDimY, int maxDimZ, int maxNumLocalZSticks, int maxNumLocalXYPlanes, SpfftProcessingUnitType executionUnit, int numThreads, MPI_Comm comm, SpfftExchangeType exchangeType); #endif GridInternal(const GridInternal& grid); GridInternal(GridInternal&&) = default; inline GridInternal& operator=(const GridInternal& grid) { *this = GridInternal(grid); return *this; } inline GridInternal& operator=(GridInternal&&) = default; inline auto max_dim_x() const noexcept -> int { return maxDimX_; } inline auto max_dim_y() const noexcept -> int { return maxDimY_; } inline auto max_dim_z() const noexcept -> int { return maxDimZ_; } inline auto max_num_local_z_columns() const noexcept -> int { return maxNumLocalZSticks_; } inline auto max_num_local_xy_planes() const noexcept -> int { return maxNumLocalXYPlanes_; } inline auto device_id() const noexcept -> int { return deviceId_; } inline auto num_threads() const noexcept -> int { return numThreads_; } inline auto array_host_1() -> HostArray& { return arrayHost1_; } inline auto array_host_2() -> HostArray& { return arrayHost2_; } inline auto processing_unit() -> SpfftProcessingUnitType { return executionUnit_; } inline auto local() -> bool { return isLocal_; } #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) inline auto array_gpu_1() -> GPUArray::type>& { return arrayGPU1_; } inline auto array_gpu_2() -> GPUArray::type>& { return arrayGPU2_; } inline auto fft_work_buffer() -> const std::shared_ptr>& { assert(fftWorkBuffer_); return fftWorkBuffer_; } #endif #ifdef SPFFT_MPI inline auto communicator() const -> const MPICommunicatorHandle& { return comm_; } inline auto exchange_type() const -> SpfftExchangeType { return exchangeType_; } #endif private: bool isLocal_; SpfftProcessingUnitType executionUnit_; int deviceId_, numThreads_; int maxDimX_, maxDimY_, maxDimZ_, maxNumLocalZSticks_, maxNumLocalXYPlanes_; HostArray arrayHost1_; HostArray arrayHost2_; #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) GPUArray::type> arrayGPU1_; GPUArray::type> arrayGPU2_; std::shared_ptr> fftWorkBuffer_; #endif #ifdef SPFFT_MPI MPICommunicatorHandle comm_; SpfftExchangeType exchangeType_; #endif }; } // namespace spfft #endif SpFFT-0.9.10/src/spfft/multi_transform.cpp000066400000000000000000000072201357247005600203770ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/multi_transform.h" #include #include "spfft/config.h" #include "spfft/multi_transform.hpp" #include "spfft/multi_transform_internal.hpp" #include "spfft/types.h" namespace spfft { void multi_transform_forward(int numTransforms, Transform* transforms, SpfftProcessingUnitType* inputLocations, double** outputPointers, SpfftScalingType* scalingTypes) { MultiTransformInternal::forward(numTransforms, transforms, inputLocations, outputPointers, scalingTypes); } void multi_transform_backward(int numTransforms, Transform* transforms, double** inputPointers, SpfftProcessingUnitType* outputLocations) { MultiTransformInternal::backward(numTransforms, transforms, inputPointers, outputLocations); } } // namespace spfft extern "C" { SpfftError spfft_multi_transform_forward(int numTransforms, SpfftTransform* transforms, SpfftProcessingUnitType* inputLocations, double** outputPointers, SpfftScalingType* scalingTypes) { try { multi_transform_forward(numTransforms, reinterpret_cast(transforms), inputLocations, outputPointers, scalingTypes); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_multi_transform_backward(int numTransforms, SpfftTransform* transforms, double** inputPointers, SpfftProcessingUnitType* outputLocations) { try { multi_transform_backward(numTransforms, reinterpret_cast(transforms), inputPointers, outputLocations); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } } SpFFT-0.9.10/src/spfft/multi_transform_float.cpp000066400000000000000000000074621357247005600215740ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/multi_transform_float.h" #include "spfft/config.h" #include "spfft/multi_transform_float.hpp" #include "spfft/multi_transform_internal.hpp" #include "spfft/types.h" namespace spfft { #ifdef SPFFT_SINGLE_PRECISION void multi_transform_forward(int numTransforms, TransformFloat* transforms, SpfftProcessingUnitType* inputLocations, float** outputPointers, SpfftScalingType* scalingTypes) { MultiTransformInternal::forward(numTransforms, transforms, inputLocations, outputPointers, scalingTypes); } void multi_transform_backward(int numTransforms, TransformFloat* transforms, float** inputPointers, SpfftProcessingUnitType* outputLocations) { MultiTransformInternal::backward(numTransforms, transforms, inputPointers, outputLocations); } #endif } // namespace spfft extern "C" { SpfftError spfft_float_multi_transform_forward(int numTransforms, SpfftFloatTransform* transforms, SpfftProcessingUnitType* inputLocations, float** outputPointers, SpfftScalingType* scalingTypes) { try { multi_transform_forward(numTransforms, reinterpret_cast(transforms), inputLocations, outputPointers, scalingTypes); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_multi_transform_backward(int numTransforms, SpfftFloatTransform* transforms, float** inputPointers, SpfftProcessingUnitType* outputLocations) { try { multi_transform_backward(numTransforms, reinterpret_cast(transforms), inputPointers, outputLocations); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } } SpFFT-0.9.10/src/spfft/multi_transform_internal.hpp000066400000000000000000000130071357247005600223000ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_MULTI_TRANSFORM_INTERNAL_HPP #define SPFFT_MULTI_TRANSFORM_INTERNAL_HPP #include "spfft/exceptions.hpp" #include "spfft/transform.hpp" #include "spfft/transform_internal.hpp" #include "timing/timing.hpp" #ifdef SPFFT_SINGLE_PRECISION #include "spfft/transform_float.hpp" #endif namespace spfft { template class MultiTransformInternal { public: using ValueType = typename TransformType::ValueType; inline static auto forward(const int numTransforms, TransformType* transforms, SpfftProcessingUnitType* inputLocations, ValueType** outputPointers, SpfftScalingType* scalingTypes) -> void { HOST_TIMING_SCOPED("forward") // transforms must not share grids for (int t1 = 0; t1 < numTransforms; ++t1) { for (int t2 = t1 + 1; t2 < numTransforms; ++t2) { if (transforms[t1].transform_->shared_grid(*(transforms[t2].transform_))) { throw InvalidParameterError(); } } } // launch all gpu transforms first for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() == SPFFT_PU_GPU) { transforms[t].transform_->forward_xy(inputLocations[t]); } } // launch all cpu transforms including MPI exchange for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() != SPFFT_PU_GPU) { transforms[t].transform_->forward_xy(inputLocations[t]); transforms[t].transform_->forward_exchange(); } } // launch all GPU MPI exhanges and transform for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() == SPFFT_PU_GPU) { transforms[t].transform_->forward_exchange(); transforms[t].transform_->forward_z(outputPointers[t], scalingTypes[t]); } } // launch all remaining cpu transforms for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() != SPFFT_PU_GPU) { transforms[t].transform_->forward_z(outputPointers[t], scalingTypes[t]); } } // synchronize all transforms for (int t = 0; t < numTransforms; ++t) { transforms[t].transform_->synchronize(); } } inline static auto backward(const int numTransforms, TransformType* transforms, ValueType** inputPointers, SpfftProcessingUnitType* outputLocations) -> void { HOST_TIMING_SCOPED("backward") // transforms must not share grids for (int t1 = 0; t1 < numTransforms; ++t1) { for (int t2 = t1 + 1; t2 < numTransforms; ++t2) { if (transforms[t1].transform_->shared_grid(*(transforms[t2].transform_))) { throw InvalidParameterError(); } } } // launch all gpu transforms first for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() == SPFFT_PU_GPU) { transforms[t].transform_->backward_z(inputPointers[t]); } } // launch all cpu transforms including MPI exchange for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() != SPFFT_PU_GPU) { transforms[t].transform_->backward_z(inputPointers[t]); transforms[t].transform_->backward_exchange(); } } // launch all GPU MPI exhanges and transform for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() == SPFFT_PU_GPU) { transforms[t].transform_->backward_exchange(); transforms[t].transform_->backward_xy(outputLocations[t]); } } // launch all remaining cpu transforms for (int t = 0; t < numTransforms; ++t) { if (transforms[t].transform_->processing_unit() != SPFFT_PU_GPU) { transforms[t].transform_->backward_xy(outputLocations[t]); } } // synchronize all transforms for (int t = 0; t < numTransforms; ++t) { transforms[t].transform_->synchronize(); } } }; } // namespace spfft #endif SpFFT-0.9.10/src/spfft/transform.cpp000066400000000000000000000324731357247005600171750ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/transform.hpp" #include "parameters/parameters.hpp" #include "spfft/grid.hpp" #include "spfft/transform.h" #include "spfft/transform_internal.hpp" namespace spfft { //--------------------- // Double precision //--------------------- Transform::Transform(const std::shared_ptr>& grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) { if (dimX < 0 || dimY < 0 || dimZ < 0 || localZLength < 0 || numLocalElements < 0 || (!indices && numLocalElements > 0)) { throw InvalidParameterError(); } std::shared_ptr param; if (!grid->local()) { #ifdef SPFFT_MPI param.reset(new Parameters(grid->communicator(), transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices)); #else throw MPISupportError(); #endif } else { param.reset( new Parameters(transformType, dimX, dimY, dimZ, numLocalElements, indexFormat, indices)); } transform_.reset(new TransformInternal(processingUnit, grid, std::move(param))); } Transform::Transform(std::shared_ptr> transform) : transform_(std::move(transform)) {} Transform Transform::clone() const { return Transform(std::shared_ptr>( new TransformInternal(transform_->clone()))); } double* Transform::space_domain_data(SpfftProcessingUnitType dataLocation) { return transform_->space_domain_data(dataLocation); } void Transform::forward(SpfftProcessingUnitType inputLocation, double* output, SpfftScalingType scaling) { transform_->forward(inputLocation, output, scaling); } void Transform::backward(const double* input, SpfftProcessingUnitType outputLocation) { transform_->backward(input, outputLocation); } SpfftTransformType Transform::type() const { return transform_->type(); } int Transform::dim_x() const { return transform_->dim_x(); } int Transform::dim_y() const { return transform_->dim_y(); } int Transform::dim_z() const { return transform_->dim_z(); } int Transform::local_z_length() const { return transform_->num_local_xy_planes(); } int Transform::local_z_offset() const { return transform_->local_xy_plane_offset(); } int Transform::local_slice_size() const { return dim_x() * dim_y() * local_z_length(); } int Transform::num_local_elements() const { return transform_->num_local_elements(); } long long int Transform::num_global_elements() const { return transform_->num_global_elements(); } long long int Transform::global_size() const { return transform_->global_size(); } SpfftProcessingUnitType Transform::processing_unit() const { return transform_->processing_unit(); } int Transform::device_id() const { return transform_->device_id(); } int Transform::num_threads() const { return transform_->num_threads(); } #ifdef SPFFT_MPI MPI_Comm Transform::communicator() const { return transform_->communicator(); } #endif } // namespace spfft //--------------------- // C API //--------------------- extern "C" { SpfftError spfft_transform_create(SpfftTransform* transform, SpfftGrid grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) { try { *transform = new spfft::Transform(reinterpret_cast(grid)->create_transform( processingUnit, transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices)); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_destroy(SpfftTransform transform) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { delete reinterpret_cast(transform); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } transform = nullptr; return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_clone(SpfftTransform transform, SpfftTransform* newTransform) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *newTransform = new spfft::Transform(reinterpret_cast(transform)->clone()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_forward(SpfftTransform transform, SpfftProcessingUnitType inputLocation, double* output, SpfftScalingType scaling) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { reinterpret_cast(transform)->forward(inputLocation, output, scaling); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_backward(SpfftTransform transform, const double* input, SpfftProcessingUnitType outputLocation) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { reinterpret_cast(transform)->backward(input, outputLocation); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_get_space_domain(SpfftTransform transform, SpfftProcessingUnitType dataLocation, double** data) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *data = reinterpret_cast(transform)->space_domain_data(dataLocation); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_dim_x(SpfftTransform transform, int* dimX) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimX = reinterpret_cast(transform)->dim_x(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_dim_y(SpfftTransform transform, int* dimY) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimY = reinterpret_cast(transform)->dim_y(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_dim_z(SpfftTransform transform, int* dimZ) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimZ = reinterpret_cast(transform)->dim_z(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_local_z_length(SpfftTransform transform, int* localZLength) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *localZLength = reinterpret_cast(transform)->local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_local_slice_size(SpfftTransform transform, int* size) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *size = reinterpret_cast(transform)->local_slice_size(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_local_z_offset(SpfftTransform transform, int* offset) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *offset = reinterpret_cast(transform)->local_z_offset(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_num_local_elements(SpfftTransform transform, int* localZLength) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *localZLength = reinterpret_cast(transform)->local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_num_global_elements(SpfftTransform transform, long long int* numGlobalElements) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numGlobalElements = reinterpret_cast(transform)->num_global_elements(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_global_size(SpfftTransform transform, long long int* globalSize) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *globalSize = reinterpret_cast(transform)->global_size(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_device_id(SpfftTransform transform, int* deviceId) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *deviceId = reinterpret_cast(transform)->device_id(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_transform_num_threads(SpfftTransform transform, int* numThreads) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numThreads = reinterpret_cast(transform)->num_threads(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_transform_communicator(SpfftTransform transform, MPI_Comm* comm) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *comm = reinterpret_cast(transform)->communicator(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_transform_communicator_fortran(SpfftGrid grid, int* commFortran) { if (!grid) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *commFortran = MPI_Comm_c2f(reinterpret_cast(grid)->communicator()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif } // extern C SpFFT-0.9.10/src/spfft/transform_float.cpp000066400000000000000000000337401357247005600203600ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/transform_float.hpp" #include "spfft/grid_float.hpp" #include "spfft/transform_float.h" #include "spfft/transform_internal.hpp" #ifdef SPFFT_SINGLE_PRECISION namespace spfft { TransformFloat::TransformFloat(const std::shared_ptr>& grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) { std::shared_ptr param; if (!grid->local()) { #ifdef SPFFT_MPI param.reset(new Parameters(grid->communicator(), transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices)); #else throw MPISupportError(); #endif } else { param.reset( new Parameters(transformType, dimX, dimY, dimZ, numLocalElements, indexFormat, indices)); } transform_.reset(new TransformInternal(processingUnit, grid, std::move(param))); } TransformFloat::TransformFloat(std::shared_ptr> transform) : transform_(std::move(transform)) {} TransformFloat TransformFloat::clone() const { return TransformFloat( std::shared_ptr>(new TransformInternal(transform_->clone()))); } float* TransformFloat::space_domain_data(SpfftProcessingUnitType dataLocation) { return transform_->space_domain_data(dataLocation); } void TransformFloat::forward(SpfftProcessingUnitType inputLocation, float* output, SpfftScalingType scaling) { transform_->forward(inputLocation, output, scaling); } void TransformFloat::backward(const float* input, SpfftProcessingUnitType outputLocation) { transform_->backward(input, outputLocation); } SpfftTransformType TransformFloat::type() const { return transform_->type(); } int TransformFloat::dim_x() const { return transform_->dim_x(); } int TransformFloat::dim_y() const { return transform_->dim_y(); } int TransformFloat::dim_z() const { return transform_->dim_z(); } int TransformFloat::local_z_length() const { return transform_->num_local_xy_planes(); } int TransformFloat::local_z_offset() const { return transform_->local_xy_plane_offset(); } int TransformFloat::local_slice_size() const { return dim_x() * dim_y() * local_z_length(); } int TransformFloat::num_local_elements() const { return transform_->num_local_elements(); } long long int TransformFloat::num_global_elements() const { return transform_->num_global_elements(); } long long int TransformFloat::global_size() const { return transform_->global_size(); } SpfftProcessingUnitType TransformFloat::processing_unit() const { return transform_->processing_unit(); } int TransformFloat::device_id() const { return transform_->device_id(); } int TransformFloat::num_threads() const { return transform_->num_threads(); } #ifdef SPFFT_MPI MPI_Comm TransformFloat::communicator() const { return transform_->communicator(); } #endif } // namespace spfft //--------------------- // C API //--------------------- extern "C" { SpfftError spfft_float_transform_create(SpfftFloatTransform* transform, SpfftFloatGrid grid, SpfftProcessingUnitType processingUnit, SpfftTransformType transformType, int dimX, int dimY, int dimZ, int localZLength, int numLocalElements, SpfftIndexFormatType indexFormat, const int* indices) { try { *transform = new spfft::TransformFloat(reinterpret_cast(grid)->create_transform( processingUnit, transformType, dimX, dimY, dimZ, localZLength, numLocalElements, indexFormat, indices)); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_destroy(SpfftFloatTransform transform) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { delete reinterpret_cast(transform); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } transform = nullptr; return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_clone(SpfftFloatTransform transform, SpfftFloatTransform* newTransform) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *newTransform = new spfft::TransformFloat(reinterpret_cast(transform)->clone()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_forward(SpfftFloatTransform transform, SpfftProcessingUnitType inputLocation, float* output, SpfftScalingType scaling) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { reinterpret_cast(transform)->forward(inputLocation, output, scaling); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_backward(SpfftFloatTransform transform, const float* input, SpfftProcessingUnitType outputLocation) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { reinterpret_cast(transform)->backward(input, outputLocation); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_get_space_domain(SpfftFloatTransform transform, SpfftProcessingUnitType dataLocation, float** data) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *data = reinterpret_cast(transform)->space_domain_data(dataLocation); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_dim_x(SpfftFloatTransform transform, int* dimX) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimX = reinterpret_cast(transform)->dim_x(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_dim_y(SpfftFloatTransform transform, int* dimY) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimY = reinterpret_cast(transform)->dim_y(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_dim_z(SpfftFloatTransform transform, int* dimZ) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *dimZ = reinterpret_cast(transform)->dim_z(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_local_z_length(SpfftFloatTransform transform, int* localZLength) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *localZLength = reinterpret_cast(transform)->local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_local_z_offset(SpfftFloatTransform transform, int* offset) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *offset = reinterpret_cast(transform)->local_z_offset(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_local_slice_size(SpfftFloatTransform transform, int* size) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *size = reinterpret_cast(transform)->local_slice_size(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_num_local_elements(SpfftFloatTransform transform, int* localZLength) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *localZLength = reinterpret_cast(transform)->local_z_length(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_num_global_elements(SpfftFloatTransform transform, long long int* numGlobalElements) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numGlobalElements = reinterpret_cast(transform)->num_global_elements(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_global_size(SpfftFloatTransform transform, long long int* globalSize) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *globalSize = reinterpret_cast(transform)->global_size(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_device_id(SpfftFloatTransform transform, int* deviceId) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *deviceId = reinterpret_cast(transform)->device_id(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SpfftError spfft_float_transform_num_threads(SpfftFloatTransform transform, int* numThreads) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *numThreads = reinterpret_cast(transform)->num_threads(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #ifdef SPFFT_MPI SpfftError spfft_float_transform_communicator(SpfftFloatTransform transform, MPI_Comm* comm) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *comm = reinterpret_cast(transform)->communicator(); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } SPFFT_EXPORT SpfftError spfft_float_transform_communicator_fortran(SpfftFloatTransform transform, int* commFortran) { if (!transform) { return SpfftError::SPFFT_INVALID_HANDLE_ERROR; } try { *commFortran = MPI_Comm_c2f(reinterpret_cast(transform)->communicator()); } catch (const spfft::GenericError& e) { return e.error_code(); } catch (...) { return SpfftError::SPFFT_UNKNOWN_ERROR; } return SpfftError::SPFFT_SUCCESS; } #endif } // extern C #endif SpFFT-0.9.10/src/spfft/transform_internal.cpp000066400000000000000000000264371357247005600210740ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include "compression/indices.hpp" #include "execution/execution_host.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "spfft/transform_internal.hpp" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "gpu_util/gpu_device_guard.hpp" #include "gpu_util/gpu_transfer.hpp" #endif namespace spfft { template TransformInternal::TransformInternal(SpfftProcessingUnitType executionUnit, std::shared_ptr> grid, std::shared_ptr param) : executionUnit_(executionUnit), param_(std::move(param)), grid_(std::move(grid)) { // ---------------------- // Input Check // ---------------------- if (!grid_) { throw InvalidParameterError(); } if (param_->local_num_xy_planes() > static_cast(grid_->max_num_local_xy_planes())) { throw InvalidParameterError(); } if (grid_->local() && param_->dim_z() != param_->local_num_xy_planes()) { throw InvalidParameterError(); } if (param_->local_num_z_sticks() > static_cast(grid_->max_num_local_z_columns())) { throw InvalidParameterError(); } if (param_->dim_x() > static_cast(grid_->max_dim_x()) || param_->dim_y() > static_cast(grid_->max_dim_y()) || param_->dim_z() > static_cast(grid_->max_dim_z())) { throw InvalidParameterError(); } if (!(executionUnit & grid_->processing_unit())) { // must match memory initialization parameters for grid throw InvalidParameterError(); } if (executionUnit != SpfftProcessingUnitType::SPFFT_PU_HOST && executionUnit != SpfftProcessingUnitType::SPFFT_PU_GPU) { // must be exclusively CPU or GPU throw InvalidParameterError(); } #ifdef SPFFT_MPI if (grid_->communicator().size() != param_->comm_size() || grid_->communicator().rank() != param_->comm_rank()) { throw InternalError(); } #endif // create execution if (grid_->local()) { // ---------------------- // Local // ---------------------- if (executionUnit == SpfftProcessingUnitType::SPFFT_PU_HOST) { execHost_.reset(new ExecutionHost(grid_->num_threads(), param_, grid_->array_host_1(), grid_->array_host_2())); } else { // GPU #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_.reset(new ExecutionGPU(grid_->num_threads(), param_, grid_->array_host_1(), grid_->array_host_2(), grid_->array_gpu_1(), grid_->array_gpu_2(), grid_->fft_work_buffer())); #else throw GPUSupportError(); #endif } } else { // ---------------------- // Distributed // ---------------------- #ifdef SPFFT_MPI if (executionUnit == SpfftProcessingUnitType::SPFFT_PU_HOST) { // CPU execHost_.reset(new ExecutionHost(grid_->communicator(), grid_->exchange_type(), grid_->num_threads(), param_, grid_->array_host_1(), grid_->array_host_2())); } else { // GPU #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_.reset(new ExecutionGPU(grid_->communicator(), grid_->exchange_type(), grid_->num_threads(), param_, grid_->array_host_1(), grid_->array_host_2(), grid_->array_gpu_1(), grid_->array_gpu_2(), grid_->fft_work_buffer())); #else // GPU throw GPUSupportError(); #endif // GPU } #else // MPI throw MPISupportError(); #endif // MPI } } template auto TransformInternal::forward(const SpfftProcessingUnitType inputLocation, T* output, SpfftScalingType scaling) -> void { HOST_TIMING_SCOPED("forward") if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); if (inputLocation != SpfftProcessingUnitType::SPFFT_PU_HOST) { throw InvalidParameterError(); } execHost_->forward_xy(); execHost_->forward_exchange(false); execHost_->forward_z(output, scaling); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) assert(execGPU_); // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->forward_xy(inputLocation); execGPU_->forward_exchange(false); execGPU_->forward_z(output, scaling); execGPU_->synchronize(); #else throw GPUSupportError(); #endif } } template auto TransformInternal::clone() const -> TransformInternal { std::shared_ptr> newGrid(new GridInternal(*grid_)); return TransformInternal(executionUnit_, std::move(newGrid), param_); } template auto TransformInternal::forward_xy(const SpfftProcessingUnitType inputLocation) -> void { if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); if (inputLocation != SpfftProcessingUnitType::SPFFT_PU_HOST) { throw InvalidParameterError(); } execHost_->forward_xy(); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) assert(execGPU_); // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->forward_xy(inputLocation); #else throw GPUSupportError(); #endif } } template auto TransformInternal::forward_exchange() -> void { if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); execHost_->forward_exchange(true); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) assert(execGPU_); // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->forward_exchange(true); #else throw GPUSupportError(); #endif } } template auto TransformInternal::forward_z(T* output, SpfftScalingType scaling) -> void { if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); execHost_->forward_z(output, scaling); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) assert(execGPU_); // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->forward_z(output, scaling); #else throw GPUSupportError(); #endif } } template auto TransformInternal::backward(const T* input, const SpfftProcessingUnitType outputLocation) -> void { HOST_TIMING_SCOPED("backward") // check if input is can be accessed from gpu if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); if (outputLocation != SpfftProcessingUnitType::SPFFT_PU_HOST) { throw InvalidParameterError(); } execHost_->backward_z(input); execHost_->backward_exchange(false); execHost_->backward_xy(); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->backward_z(input); execGPU_->backward_exchange(false); execGPU_->backward_xy(outputLocation); execGPU_->synchronize(); #else throw GPUSupportError(); #endif } } template auto TransformInternal::backward_z(const T* input) -> void { // check if input is can be accessed from gpu if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); execHost_->backward_z(input); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->backward_z(input); #else throw GPUSupportError(); #endif } } template auto TransformInternal::backward_exchange() -> void { // check if input is can be accessed from gpu if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); execHost_->backward_exchange(true); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->backward_exchange(true); #else throw GPUSupportError(); #endif } } template auto TransformInternal::backward_xy(const SpfftProcessingUnitType outputLocation) -> void { // check if input is can be accessed from gpu if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_HOST) { assert(execHost_); if (outputLocation != SpfftProcessingUnitType::SPFFT_PU_HOST) { throw InvalidParameterError(); } execHost_->backward_xy(); } else { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) // set device for current thread GPUDeviceGuard(grid_->device_id()); execGPU_->backward_xy(outputLocation); #else throw GPUSupportError(); #endif } } template auto TransformInternal::space_domain_data(SpfftProcessingUnitType location) -> T* { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) if (executionUnit_ == SpfftProcessingUnitType::SPFFT_PU_GPU) { // GPU if (location == SpfftProcessingUnitType::SPFFT_PU_GPU) { return execGPU_->space_domain_data_gpu().data(); } else { return execGPU_->space_domain_data_host().data(); } } #endif // CPU if (location != SpfftProcessingUnitType::SPFFT_PU_HOST) throw InvalidParameterError(); return execHost_->space_domain_data().data(); } template auto TransformInternal::synchronize() -> void { #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) if (execGPU_) execGPU_->synchronize(); #endif } // instatiate templates for float and double template class TransformInternal; #ifdef SPFFT_SINGLE_PRECISION template class TransformInternal; #endif } // namespace spfft SpFFT-0.9.10/src/spfft/transform_internal.hpp000066400000000000000000000115261357247005600210720ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSFORM_INTERNAL_HPP #define SPFFT_TRANSFORM_INTERNAL_HPP #include #include "execution/execution_host.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/grid_internal.hpp" #include "spfft/types.h" #include "util/common_types.hpp" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "compression/compression_gpu.hpp" #include "execution/execution_gpu.hpp" #endif namespace spfft { template class TransformInternal { public: TransformInternal(SpfftProcessingUnitType executionUnit, std::shared_ptr> grid, std::shared_ptr param); auto clone() const -> TransformInternal; inline auto type() const noexcept -> SpfftTransformType { return param_->transform_type(); } inline auto dim_x() const noexcept -> int { return param_->dim_x(); } inline auto dim_y() const noexcept -> int { return param_->dim_y(); } inline auto dim_z() const noexcept -> int { return param_->dim_z(); } inline auto num_local_xy_planes() const noexcept -> int { return param_->local_num_xy_planes(); } inline auto local_xy_plane_offset() const noexcept -> int { return param_->local_xy_plane_offset(); } inline auto processing_unit() const noexcept -> SpfftProcessingUnitType { return executionUnit_; } inline auto device_id() const -> int { return grid_->device_id(); } inline auto num_threads() const -> int { return grid_->num_threads(); } inline auto num_local_elements() const -> int { return param_->local_num_elements(); } inline auto num_global_elements() const -> long long int { return param_->global_num_elements(); } inline auto global_size() const -> long long int { return param_->global_size(); } inline auto shared_grid(const TransformInternal& other) const -> bool { return other.grid_ == grid_; } inline auto transform_type() const -> SpfftTransformType { return param_->transform_type(); } #ifdef SPFFT_MPI inline auto communicator() const -> MPI_Comm { return grid_->communicator().get(); } #endif // full forward transform with blocking communication auto forward(const SpfftProcessingUnitType inputLocation, T* output, SpfftScalingType scaling) -> void; // transform in x and y auto forward_xy(const SpfftProcessingUnitType inputLocation) -> void; // start non-blocking exchange auto forward_exchange() -> void; // finalize exchange and transform z auto forward_z(T* output, SpfftScalingType scaling) -> void; // full backward transform with blocking communication auto backward(const T* input, const SpfftProcessingUnitType outputLocation) -> void; // transform in x and y auto backward_xy(const SpfftProcessingUnitType outputLocation) -> void; // start non-blocking exchange auto backward_exchange() -> void; // finalize exchange and transform z auto backward_z(const T* input) -> void; // must be called after step-wise transforms on GPUs auto synchronize() -> void; auto space_domain_data(SpfftProcessingUnitType location) -> T*; private: SpfftProcessingUnitType executionUnit_; std::shared_ptr param_; std::shared_ptr> grid_; std::unique_ptr> execHost_; #if (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) std::unique_ptr> execGPU_; #endif }; } // namespace spfft #endif SpFFT-0.9.10/src/symmetry/000077500000000000000000000000001357247005600152145ustar00rootroot00000000000000SpFFT-0.9.10/src/symmetry/gpu_kernels/000077500000000000000000000000001357247005600175325ustar00rootroot00000000000000SpFFT-0.9.10/src/symmetry/gpu_kernels/symmetry_kernels.cu000066400000000000000000000161131357247005600235010ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_kernel_parameter.hpp" #include "gpu_util/gpu_runtime.hpp" #include "memory/gpu_array_const_view.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { template __global__ static void symmetrize_plane_kernel( GPUArrayView3D::type> data, const int startIndex, const int numIndices) { assert(startIndex + numIndices <= data.dim_mid()); int idxMid = threadIdx.x + blockIdx.x * blockDim.x; if (idxMid < numIndices) { idxMid += startIndex; for (int idxOuter = blockIdx.y; idxOuter < data.dim_outer(); idxOuter += gridDim.y) { auto value = data(blockIdx.y, idxMid, 0); if (value.x != T(0) || value.y != T(0)) { value.y = -value.y; data(idxOuter, data.dim_mid() - idxMid, 0) = value; } } } } auto symmetrize_plane_gpu(const gpu::StreamType stream, const GPUArrayView3D::type>& data) -> void { assert(data.size() > 2); { const int startIndex = 1; const int numIndices = data.dim_mid() / 2; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((numIndices + threadBlock.x - 1) / threadBlock.x, std::min(data.dim_outer(), gpu::GridSizeMedium)); launch_kernel(symmetrize_plane_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } { const int startIndex = data.dim_mid() / 2 + 1; const int numIndices = data.dim_mid() - startIndex; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((numIndices + threadBlock.x - 1) / threadBlock.x, std::min(data.dim_outer(), gpu::GridSizeMedium)); launch_kernel(symmetrize_plane_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } } auto symmetrize_plane_gpu(const gpu::StreamType stream, const GPUArrayView3D::type>& data) -> void { assert(data.size() > 2); { const int startIndex = 1; const int numIndices = data.dim_mid() / 2; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((numIndices + threadBlock.x - 1) / threadBlock.x, std::min(data.dim_outer(), gpu::GridSizeMedium)); launch_kernel(symmetrize_plane_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } { const int startIndex = data.dim_mid() / 2 + 1; const int numIndices = data.dim_mid() - startIndex; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((numIndices + threadBlock.x - 1) / threadBlock.x, std::min(data.dim_outer(), gpu::GridSizeMedium)); launch_kernel(symmetrize_plane_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } } template __global__ static void symmetrize_stick_kernel( GPUArrayView1D::type> data, const int startIndex, const int numIndices) { assert(startIndex + numIndices <= data.size()); for (int idxInner = threadIdx.x + blockIdx.x * blockDim.x + startIndex; idxInner < numIndices + startIndex; idxInner += gridDim.x * blockDim.x) { auto value = data(idxInner); if (value.x != T(0) || value.y != T(0)) { value.y = -value.y; data(data.size() - idxInner) = value; } } } auto symmetrize_stick_gpu(const gpu::StreamType stream, const GPUArrayView1D::type>& data) -> void { assert(data.size() > 2); { const int startIndex = 1; const int numIndices = data.size() / 2; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid(std::min( static_cast((numIndices + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); launch_kernel(symmetrize_stick_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } { const int startIndex = data.size() / 2 + 1; const int numIndices = data.size() - startIndex; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid(std::min( static_cast((numIndices + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); launch_kernel(symmetrize_stick_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } } auto symmetrize_stick_gpu(const gpu::StreamType stream, const GPUArrayView1D::type>& data) -> void { assert(data.size() > 2); { const int startIndex = 1; const int numIndices = data.size() / 2; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid(std::min( static_cast((numIndices + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); launch_kernel(symmetrize_stick_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } { const int startIndex = data.size() / 2 + 1; const int numIndices = data.size() - startIndex; const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid(std::min( static_cast((numIndices + threadBlock.x - 1) / threadBlock.x), gpu::GridSizeMedium)); launch_kernel(symmetrize_stick_kernel, threadGrid, threadBlock, 0, stream, data, startIndex, numIndices); } } } // namespace spfft SpFFT-0.9.10/src/symmetry/gpu_kernels/symmetry_kernels.hpp000066400000000000000000000046531357247005600236670ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SYMMETRY_KERNELS_HPP #define SPFFT_SYMMETRY_KERNELS_HPP #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_runtime_api.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { auto symmetrize_plane_gpu(const gpu::StreamType stream, const GPUArrayView3D::type>& data) -> void; auto symmetrize_plane_gpu(const gpu::StreamType stream, const GPUArrayView3D::type>& data) -> void; auto symmetrize_stick_gpu(const gpu::StreamType stream, const GPUArrayView1D::type>& data) -> void; auto symmetrize_stick_gpu(const gpu::StreamType stream, const GPUArrayView1D::type>& data) -> void; } // namespace spfft #endif SpFFT-0.9.10/src/symmetry/symmetry.hpp000066400000000000000000000033771357247005600176300ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SYMMETRY_HPP #define SPFFT_SYMMETRY_HPP #include "spfft/config.h" namespace spfft { class Symmetry { public: virtual auto apply() -> void{}; virtual ~Symmetry() = default; }; } // namespace spfft #endif SpFFT-0.9.10/src/symmetry/symmetry_gpu.hpp000066400000000000000000000061201357247005600204700ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SYMMETRY_GPU_HPP #define SPFFT_SYMMETRY_GPU_HPP #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array_view.hpp" #include "spfft/config.h" #include "symmetry/gpu_kernels/symmetry_kernels.hpp" #include "symmetry/symmetry.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" namespace spfft { // This class will apply the 1D hermitian symmetry along the inner dimension on the plane with mid // index 0 template class PlaneSymmetryGPU : public Symmetry { public: PlaneSymmetryGPU(GPUStreamHandle stream, const GPUArrayView3D::type>& data) : stream_(std::move(stream)), data_(data) {} auto apply() -> void override { if (data_.dim_mid() > 2 && data_.size() > 0) { symmetrize_plane_gpu(stream_.get(), data_); } } private: GPUStreamHandle stream_; GPUArrayView3D::type> data_; }; // This class will apply the hermitian symmetry in 1d template class StickSymmetryGPU : public Symmetry { public: StickSymmetryGPU(GPUStreamHandle stream, const GPUArrayView1D::type>& stick) : stream_(std::move(stream)), stick_(stick) {} auto apply() -> void override { if (stick_.size() > 2) { symmetrize_stick_gpu(stream_.get(), stick_); } } private: GPUStreamHandle stream_; GPUArrayView1D::type> stick_; }; } // namespace spfft #endif SpFFT-0.9.10/src/symmetry/symmetry_host.hpp000066400000000000000000000074501357247005600206610ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_SYMMETRY_HOST_HPP #define SPFFT_SYMMETRY_HOST_HPP #include #include "memory/host_array_view.hpp" #include "spfft/config.h" #include "symmetry/symmetry.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" namespace spfft { // This class will apply the 1D hermitian symmetry along the inner dimension on the plane with mid // index 0 template class PlaneSymmetryHost : public Symmetry { public: explicit PlaneSymmetryHost(const HostArrayView3D>& data) : data_(data) {} auto apply() -> void override { constexpr std::complex zeroElement; // Data may be conjugated twice, but this way symmetry is applied independent of positive or // negative frequencies provided SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType idxOuter = 0; idxOuter < data_.dim_outer(); ++idxOuter) { for (SizeType idxInner = 1; idxInner < data_.dim_inner(); ++idxInner) { const auto value = data_(idxOuter, 0, idxInner); if (value != zeroElement) { data_(idxOuter, 0, data_.dim_inner() - idxInner) = std::conj(value); } } } } private: HostArrayView3D> data_; }; // This class will apply the hermitian symmetry in 1d template class StickSymmetryHost : public Symmetry { public: explicit StickSymmetryHost(const HostArrayView1D>& stick) : stick_(stick) {} auto apply() -> void override { constexpr std::complex zeroElement; // Data may be conjugated twice, but this way symmetry is applied independent of positive or // negative frequencies provided SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType idxInner = 1; idxInner < stick_.size() / 2 + 1; ++idxInner) { const auto value = stick_(idxInner); if (value != zeroElement) { stick_(stick_.size() - idxInner) = std::conj(value); } } SPFFT_OMP_PRAGMA("omp for schedule(static)") for (SizeType idxInner = stick_.size() / 2 + 1; idxInner < stick_.size(); ++idxInner) { const auto value = stick_(idxInner); if (value != zeroElement) { stick_(stick_.size() - idxInner) = std::conj(value); } } } private: HostArrayView1D> stick_; }; } // namespace spfft #endif SpFFT-0.9.10/src/timing/000077500000000000000000000000001357247005600146125ustar00rootroot00000000000000SpFFT-0.9.10/src/timing/rt_graph.cpp000066400000000000000000000437421357247005600171360ustar00rootroot00000000000000/* * Copyright (c) 2019 Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "timing/rt_graph.hpp" #include #include #include #include #include #include #include #include #include namespace rt_graph { // ====================== // internal helper // ====================== namespace internal { namespace { struct Format { Format(Stat stat_) : stat(stat_) { switch (stat_) { case Stat::Count: header = "#"; space = 6; break; case Stat::Total: header = "Total"; space = 14; break; case Stat::Mean: header = "Mean"; space = 14; break; case Stat::Median: header = "Median"; space = 14; break; case Stat::QuartileHigh: header = "Quartile High"; space = 14; break; case Stat::QuartileLow: header = "Quartile Low"; space = 14; break; case Stat::Min: header = "Min"; space = 14; break; case Stat::Max: header = "Max"; space = 14; break; case Stat::Percentage: header = "%"; space = 11; break; case Stat::ParentPercentage: header = "Parent %"; space = 11; break; } } Stat stat; std::string header; std::size_t space; }; // format time input in seconds into string with appropriate unit auto format_time(const double time_seconds) -> std::string { if (time_seconds <= 0.0) return std::string("0 s"); // time is always greater than 0 here const double exponent = std::log10(std::abs(time_seconds)); const int siExponent = static_cast(std::floor(exponent / 3.0) * 3); std::stringstream result; result << std::fixed << std::setprecision(2); result << time_seconds * std::pow(10.0, static_cast(-siExponent)); result << " "; switch (siExponent) { case 24: result << "Y"; break; case 21: result << "Z"; break; case 18: result << "E"; break; case 15: result << "P"; break; case 12: result << "T"; break; case 9: result << "G"; break; case 6: result << "M"; break; case 3: result << "k"; break; case 0: break; case -3: result << "m"; break; case -6: result << "u"; break; case -9: result << "n"; break; case -12: result << "p"; break; case -15: result << "f"; break; case -18: result << "a"; break; case -21: result << "z"; break; case -24: result << "y"; break; default: result << "?"; } result << "s"; return result.str(); } auto calc_median(const std::vector::const_iterator& begin, const std::vector::const_iterator& end) -> double { const auto n = end - begin; if (n == 0) return 0.0; if (n % 2 == 0) { return (*(begin + n / 2) + *(begin + n / 2 - 1)) / 2.0; } else { return *(begin + n / 2); } } auto print_stat(std::ostream& out, const Format& format, const std::vector& sortedTimings, double totalSum, double parentSum, double currentSum) -> void { switch (format.stat) { case Stat::Count: out << std::right << std::setw(format.space) << sortedTimings.size(); break; case Stat::Total: out << std::right << std::setw(format.space) << format_time(currentSum); break; case Stat::Mean: out << std::right << std::setw(format.space) << format_time(currentSum / sortedTimings.size()); break; case Stat::Median: out << std::right << std::setw(format.space) << format_time(calc_median(sortedTimings.begin(), sortedTimings.end())); break; case Stat::QuartileHigh: { const double upperQuartile = calc_median(sortedTimings.begin() + sortedTimings.size() / 2 + (sortedTimings.size() % 2) * (sortedTimings.size() > 1), sortedTimings.end()); out << std::right << std::setw(format.space) << format_time(upperQuartile); } break; case Stat::QuartileLow: { const double lowerQuartile = calc_median(sortedTimings.begin(), sortedTimings.begin() + sortedTimings.size() / 2); out << std::right << std::setw(format.space) << format_time(lowerQuartile); } break; case Stat::Min: out << std::right << std::setw(format.space) << format_time(sortedTimings.front()); break; case Stat::Max: out << std::right << std::setw(format.space) << format_time(sortedTimings.back()); break; case Stat::Percentage: { const double p = (totalSum < currentSum || totalSum == 0) ? 100.0 : currentSum / totalSum * 100.0; out << std::right << std::fixed << std::setprecision(2) << std::setw(format.space) << p; } break; case Stat::ParentPercentage: { const double p = (parentSum < currentSum || parentSum == 0) ? 100.0 : currentSum / parentSum * 100.0; out << std::right << std::fixed << std::setprecision(2) << std::setw(format.space) << p; } break; } } // Helper struct for creating a tree of timings struct TimeStampPair { std::string identifier; double time = 0.0; std::size_t startIdx = 0; std::size_t stopIdx = 0; internal::TimingNode* nodePtr = nullptr; }; auto calculate_statistic(std::vector values) -> std::tuple { if (values.empty()) return std::make_tuple(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); std::sort(values.begin(), values.end()); const double min = values.front(); const double max = values.back(); const double median = calc_median(values.begin(), values.end()); const double sum = std::accumulate(values.begin(), values.end(), 0.0); const double mean = sum / values.size(); const double lowerQuartile = calc_median(values.begin(), values.begin() + values.size() / 2); const double upperQuartile = calc_median( values.begin() + values.size() / 2 + (values.size() % 2) * (values.size() > 1), values.end()); return std::make_tuple(sum, mean, median, min, max, lowerQuartile, upperQuartile); } // print rt_graph nodes in tree recursively auto print_node(std::ostream& out, const std::vector formats, const std::size_t identifierSpace, const std::string& nodePrefix, const internal::TimingNode& node, const bool isSubNode, const bool isLastSubnode, double parentTime, double totalTime) -> void { double sum, mean, median, min, max, lowerQuartile, upperQuartile; std::tie(sum, mean, median, min, max, lowerQuartile, upperQuartile) = calculate_statistic(node.timings); if (!isSubNode) { totalTime = sum; parentTime = sum; } const double totalPercentage = (totalTime < sum || totalTime == 0) ? 100.0 : sum / totalTime * 100.0; const double parentPercentage = (parentTime < sum || parentTime == 0) ? 100.0 : sum / parentTime * 100.0; std::stringstream totalPercentageStream; totalPercentageStream << std::fixed << std::setprecision(2) << totalPercentage; std::stringstream parentPercentageStream; parentPercentageStream << std::fixed << std::setprecision(2) << parentPercentage; out << std::left << std::setw(identifierSpace); if (isSubNode) out << nodePrefix + "- " + node.identifier; else out << nodePrefix + node.identifier; auto sortedTimings = node.timings; std::sort(sortedTimings.begin(), sortedTimings.end()); const double currentTime = std::accumulate(sortedTimings.begin(), sortedTimings.end(), 0.0); for (const auto& format : formats) { print_stat(out, format, sortedTimings, totalTime, parentTime, currentTime); } out << std::endl; for (const auto& subNode : node.subNodes) { print_node(out, formats, identifierSpace, nodePrefix + std::string(" |"), subNode, true, &subNode == &node.subNodes.back(), sum, totalTime); if (!isLastSubnode && &subNode == &node.subNodes.back()) { out << nodePrefix << std::endl; } } } // determine length of padding required for printing entire tree identifiers recursively auto max_node_identifier_length(const internal::TimingNode& node, const std::size_t recursionDepth, const std::size_t addPerLevel, const std::size_t parentMax) -> std::size_t { std::size_t currentLength = node.identifier.length() + recursionDepth * addPerLevel; std::size_t max = currentLength > parentMax ? currentLength : parentMax; for (const auto& subNode : node.subNodes) { const std::size_t subMax = max_node_identifier_length(subNode, recursionDepth + 1, addPerLevel, max); if (subMax > max) max = subMax; } return max; } auto export_node_json(const std::string& padding, const std::list& nodeList, std::ostream& stream) -> void { stream << "{" << std::endl; const std::string nodePadding = padding + " "; const std::string subNodePadding = nodePadding + " "; for (const auto& node : nodeList) { stream << nodePadding << "\"" << node.identifier << "\" : {" << std::endl; stream << subNodePadding << "\"timings\" : ["; for (const auto& value : node.timings) { stream << value; if (&value != &(node.timings.back())) stream << ", "; } stream << "]," << std::endl; stream << subNodePadding << "\"sub-timings\" : "; export_node_json(subNodePadding, node.subNodes, stream); stream << nodePadding << "}"; if (&node != &(nodeList.back())) stream << ","; stream << std::endl; } stream << padding << "}" << std::endl; } auto extract_timings(const std::string& identifier, const std::list& nodes, std::vector& timings) -> void { for (const auto& node : nodes) { if (node.identifier == identifier) { timings.insert(timings.end(), node.timings.begin(), node.timings.end()); } extract_timings(identifier, node.subNodes, timings); } } } // namespace } // namespace internal // ====================== // Timer // ====================== auto Timer::process() const -> TimingResult { std::list results; std::stringstream warnings; try { std::vector timePairs; timePairs.reserve(timeStamps_.size() / 2); // create pairs of start / stop timings for (std::size_t i = 0; i < timeStamps_.size(); ++i) { if (timeStamps_[i].type == internal::TimeStampType::Start) { internal::TimeStampPair pair; pair.startIdx = i; pair.identifier = std::string(timeStamps_[i].identifierPtr); std::size_t numInnerMatchingIdentifiers = 0; // search for matching stop after start for (std::size_t j = i + 1; j < timeStamps_.size(); ++j) { // only consider matching identifiers if (std::string(timeStamps_[j].identifierPtr) == std::string(timeStamps_[i].identifierPtr)) { if (timeStamps_[j].type == internal::TimeStampType::Stop && numInnerMatchingIdentifiers == 0) { // Matching stop found std::chrono::duration duration = timeStamps_[j].time - timeStamps_[i].time; pair.time = duration.count(); pair.stopIdx = j; timePairs.push_back(pair); if (pair.time < 0) { warnings << "rt_graph WARNING:Measured time is negative. Non-steady system-clock?!" << std::endl; } break; } else if (timeStamps_[j].type == internal::TimeStampType::Stop && numInnerMatchingIdentifiers > 0) { // inner stop with matching identifier --numInnerMatchingIdentifiers; } else if (timeStamps_[j].type == internal::TimeStampType::Start) { // inner start with matching identifier ++numInnerMatchingIdentifiers; } } } if (pair.stopIdx == 0) { warnings << "rt_graph WARNING: Start / stop time stamps do not match for \"" << timeStamps_[i].identifierPtr << "\"!" << std::endl; } } } // create tree of timings where sub-nodes represent timings fully enclosed by another start / // stop pair Use the fact that timePairs is sorted by startIdx for (std::size_t i = 0; i < timePairs.size(); ++i) { auto& pair = timePairs[i]; // find potential parent by going backwards through pairs, starting with the current pair // position for (auto timePairIt = timePairs.rbegin() + (timePairs.size() - i); timePairIt != timePairs.rend(); ++timePairIt) { if (timePairIt->stopIdx > pair.stopIdx && timePairIt->nodePtr != nullptr) { auto& parentNode = *(timePairIt->nodePtr); // check if sub-node with identifier exists bool nodeFound = false; for (auto& subNode : parentNode.subNodes) { if (subNode.identifier == pair.identifier) { nodeFound = true; subNode.timings.push_back(pair.time); // mark node position in pair for finding sub-nodes pair.nodePtr = &(subNode); break; } } if (!nodeFound) { // create new sub-node internal::TimingNode newNode; newNode.identifier = pair.identifier; newNode.timings.push_back(pair.time); parentNode.subNodes.push_back(std::move(newNode)); // mark node position in pair for finding sub-nodes pair.nodePtr = &(parentNode.subNodes.back()); } break; } } // No parent found, must be top level node if (pair.nodePtr == nullptr) { // Check if top level node with same name exists for (auto& topNode : results) { if (topNode.identifier == pair.identifier) { topNode.timings.push_back(pair.time); pair.nodePtr = &(topNode); break; } } } // New top level node if (pair.nodePtr == nullptr) { internal::TimingNode newNode; newNode.identifier = pair.identifier; newNode.timings.push_back(pair.time); // newNode.parent = nullptr; results.push_back(std::move(newNode)); // mark node position in pair for finding sub-nodes pair.nodePtr = &(results.back()); } } } catch (const std::exception& e) { warnings << "rt_graph WARNING: Processing of timings failed: " << e.what() << std::endl; } catch (...) { warnings << "rt_graph WARNING: Processing of timings failed!" << std::endl; } return TimingResult(std::move(results), warnings.str()); } // ====================== // // ====================== auto TimingResult::json() const -> std::string { std::stringstream jsonStream; jsonStream << std::scientific; internal::export_node_json("", rootNodes_, jsonStream); return jsonStream.str(); } auto TimingResult::get_timings(const std::string& identifier) const -> std::vector { std::vector timings; internal::extract_timings(identifier, rootNodes_, timings); return timings; } auto TimingResult::print(std::vector statistic) const -> std::string { std::stringstream stream; // print warnings stream << warnings_; // calculate space for printing identifiers std::size_t identifierSpace = 0; for (const auto& node : rootNodes_) { const auto nodeMax = internal::max_node_identifier_length(node, 0, 2, identifierSpace); if (nodeMax > identifierSpace) identifierSpace = nodeMax; } identifierSpace += 3; auto totalSpace = identifierSpace; std::vector formats; formats.reserve(statistic.size()); for (const auto& stat : statistic) { formats.emplace_back(stat); totalSpace += formats.back().space; } // Construct table header // Table start stream << std::string(totalSpace, '=') << std::endl; // header stream << std::right << std::setw(identifierSpace) << ""; for (const auto& format : formats) { stream << std::right << std::setw(format.space) << format.header; } stream << std::endl; // Header separtion line stream << std::string(totalSpace, '-') << std::endl; // print all timings for (const auto& node : rootNodes_) { internal::print_node(stream, formats, identifierSpace, std::string(), node, false, true, 0.0, 0.0); stream << std::endl; } // End table stream << std::string(totalSpace, '=') << std::endl; return stream.str(); } } // namespace rt_graph SpFFT-0.9.10/src/timing/rt_graph.hpp000066400000000000000000000175401357247005600171400ustar00rootroot00000000000000/* * Copyright (c) 2019 Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef RT_GRAPH_HPP_GUARD #define RT_GRAPH_HPP_GUARD #include #include #include #include #include #include #include namespace rt_graph { using ClockType = std::chrono::high_resolution_clock; // Selection of available statistics enum class Stat { Count, // Number of measurements Total, // Total accumulated time Mean, // Mean time Median, // Median time QuartileHigh, // Third quartile time QuartileLow, // First quartile time Min, // Mininum time Max, // Maximum time Percentage, // Percentage of accumulated time with respect to the top-level node in graph ParentPercentage // Percentage of accumulated time with respect to the parent node in graph }; // internal helper functionality namespace internal { enum class TimeStampType { Start, Stop, Empty }; struct TimeStamp { TimeStamp() : type(TimeStampType::Empty) {} // Identifier pointer must point to compile time string literal TimeStamp(const char* identifier, const TimeStampType& stampType) : time(ClockType::now()), identifierPtr(identifier), type(stampType) {} ClockType::time_point time; const char* identifierPtr; TimeStampType type; }; struct TimingNode { std::string identifier; std::vector timings; std::list subNodes; }; } // namespace internal // Processed timings results. class TimingResult { public: TimingResult(std::list rootNodes, std::string warnings) : rootNodes_(std::move(rootNodes)), warnings_(std::move(warnings)) {} // Get json representation of the full graph with all timings. Unit of time is seconds. auto json() const -> std::string; // Get all timings for given identifier auto get_timings(const std::string& identifier) const -> std::vector; // Print graph statistic to string. auto print(std::vector statistic = {Stat::Count, Stat::Total, Stat::Percentage, Stat::ParentPercentage, Stat::Median, Stat::Min, Stat::Max}) const -> std::string; private: std::list rootNodes_; std::string warnings_; }; class ScopedTiming; // Timer class, which allows to start / stop measurements with a given identifier. class Timer { public: // reserve space for 1000'000 measurements Timer() { timeStamps_.reserve(2 * 1000 * 1000); } // reserve space for given number of measurements explicit Timer(std::size_t reserveCount) { timeStamps_.reserve(2 * reserveCount); } // start with string literal identifier template inline auto start(const char (&identifierPtr)[N]) -> void { atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering timeStamps_.emplace_back(identifierPtr, internal::TimeStampType::Start); atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering } // start with string identifier (storing string object comes with some additional overhead) inline auto start(std::string identifier) -> void { atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering identifierStrings_.emplace_back(std::move(identifier)); timeStamps_.emplace_back(identifierStrings_.back().c_str(), internal::TimeStampType::Start); atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering } // stop with string literal identifier template inline auto stop(const char (&identifierPtr)[N]) -> void { atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering timeStamps_.emplace_back(identifierPtr, internal::TimeStampType::Stop); atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering } // stop with string identifier (storing string object comes with some additional overhead) inline auto stop(std::string identifier) -> void { atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering identifierStrings_.emplace_back(std::move(identifier)); timeStamps_.emplace_back(identifierStrings_.back().c_str(), internal::TimeStampType::Stop); atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering } // clear timer and reserve space for given number of new measurements. inline auto clear(std::size_t reserveCount) -> void { timeStamps_.clear(); identifierStrings_.clear(); this->reserve(reserveCount); } // reserve space for given number of measurements. Can prevent allocations at start / stop calls. inline auto reserve(std::size_t reserveCount) -> void { timeStamps_.reserve(reserveCount); } // process timings into result type auto process() const -> TimingResult; private: inline auto stop_with_ptr(const char* identifierPtr) -> void { atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering timeStamps_.emplace_back(identifierPtr, internal::TimeStampType::Stop); atomic_signal_fence(std::memory_order_seq_cst); // only prevents compiler reordering } friend ScopedTiming; std::vector timeStamps_; std::deque identifierStrings_; // pointer to elements always remain valid after push back }; // Helper class, which calls start() upon creation and stop() on timer when leaving scope with given // identifier. class ScopedTiming { public: // timer reference must be valid for the entire lifetime template ScopedTiming(const char (&identifierPtr)[N], Timer& timer) : identifierPtr_(identifierPtr), timer_(timer) { timer_.start(identifierPtr); } ScopedTiming(std::string identifier, Timer& timer) : identifierPtr_(nullptr), identifier_(std::move(identifier)), timer_(timer) { timer_.start(identifier_); } ScopedTiming(const ScopedTiming&) = delete; ScopedTiming(ScopedTiming&&) = delete; auto operator=(const ScopedTiming&) -> ScopedTiming& = delete; auto operator=(ScopedTiming &&) -> ScopedTiming& = delete; ~ScopedTiming() { if (identifierPtr_) { timer_.stop_with_ptr(identifierPtr_); } else { timer_.stop(std::move(identifier_)); } } private: const char* identifierPtr_; std::string identifier_; Timer& timer_; }; } // namespace rt_graph #endif SpFFT-0.9.10/src/timing/timing.cpp000066400000000000000000000033111357247005600166030ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "timing/timing.hpp" namespace spfft { namespace timing { #ifdef SPFFT_TIMING ::rt_graph::Timer GlobalTimer; #endif } // namespace timing } // namespace spfft SpFFT-0.9.10/src/timing/timing.hpp000066400000000000000000000046671357247005600166270ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TIMING_HPP #define SPFFT_TIMING_HPP #include "spfft/config.h" #ifdef SPFFT_TIMING #include #include #include "timing/rt_graph.hpp" namespace spfft { namespace timing { extern ::rt_graph::Timer GlobalTimer; } // namespace timing } // namespace spfft #define HOST_TIMING_CONCAT_IMPL(x, y) x##y #define HOST_TIMING_MACRO_CONCAT(x, y) HOST_TIMING_CONCAT_IMPL(x, y) #define HOST_TIMING_SCOPED(identifier) \ ::rt_graph::ScopedTiming HOST_TIMING_MACRO_CONCAT( \ scopedHostTimerMacroGenerated, __COUNTER__)(identifier, ::spfft::timing::GlobalTimer); #define HOST_TIMING_START(identifier) ::spfft::timing::GlobalTimer.start(identifier); #define HOST_TIMING_STOP(identifier) ::spfft::timing::GlobalTimer.stop(identifier); #else #define HOST_TIMING_START(identifier) #define HOST_TIMING_STOP(identifier) #define HOST_TIMING_SCOPED(identifier) #endif #endif SpFFT-0.9.10/src/transpose/000077500000000000000000000000001357247005600153415ustar00rootroot00000000000000SpFFT-0.9.10/src/transpose/gpu_kernels/000077500000000000000000000000001357247005600176575ustar00rootroot00000000000000SpFFT-0.9.10/src/transpose/gpu_kernels/buffered_kernels.cu000066400000000000000000000406041357247005600235210ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "gpu_util/complex_conversion.cuh" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_kernel_parameter.hpp" #include "gpu_util/gpu_runtime.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array_const_view.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { // Packs z-sticks into buffer for MPI_Alltoall // Dimension of buffer are (numRanks, maxNumZSticks, maxNumXYPlanes) // Dimension of freqZData are (numLocalZSticks, dimZ) template __global__ static void buffered_pack_backward_kernel(const GPUArrayConstView1D numXYPlanes, const GPUArrayConstView1D xyPlaneOffsets, const GPUArrayConstView2D freqZData, GPUArrayView3D buffer) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; for (int r = 0; r < numXYPlanes.size(); ++r) { if (xyPlaneIndex < numXYPlanes(r)) { const int xyOffset = xyPlaneOffsets(r); for (int zStickIndex = blockIdx.y; zStickIndex < freqZData.dim_outer(); zStickIndex += gridDim.y) { buffer(r, zStickIndex, xyPlaneIndex) = ConvertComplex::apply( freqZData(zStickIndex, xyPlaneIndex + xyOffset)); } } } } template static auto buffered_pack_backward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D& freqZData, GPUArrayView3D buffer) -> void { assert(xyPlaneOffsets.size() == numXYPlanes.size()); assert(buffer.size() >= freqZData.size()); assert(buffer.dim_outer() == xyPlaneOffsets.size()); assert(buffer.dim_inner() == maxNumXYPlanes); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((maxNumXYPlanes + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); assert(threadGrid.x > 0); assert(threadGrid.y > 0); launch_kernel(buffered_pack_backward_kernel, threadGrid, threadBlock, 0, stream, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void { buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void { buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void { buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } // Unpacks z-sticks from buffer after MPI_Alltoall // Dimension of buffer are (numRanks, maxNumZSticks, maxNumXYPlanes) // Dimension of freqXYData are (numLocalXYPlanes, dimY, dimX) template __global__ static void buffered_unpack_backward_kernel( const GPUArrayConstView1D numZSticks, const GPUArrayConstView1D indices, const GPUArrayConstView3D buffer, GPUArrayView2D freqXYDataFlat) { // buffer.dim_mid() is equal to maxNumZSticks const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; if (xyPlaneIndex < freqXYDataFlat.dim_outer()) { for (int r = 0; r < numZSticks.size(); ++r) { const int numCurrentZSticks = numZSticks(r); for (int zStickIndex = blockIdx.y; zStickIndex < numCurrentZSticks; zStickIndex += gridDim.y) { const int currentIndex = indices(r * buffer.dim_mid() + zStickIndex); freqXYDataFlat(xyPlaneIndex, currentIndex) = ConvertComplex::apply(buffer(r, zStickIndex, xyPlaneIndex)); } } } } template static auto buffered_unpack_backward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D& buffer, GPUArrayView3D freqXYData) -> void { assert(buffer.dim_outer() == numZSticks.size()); assert(buffer.dim_inner() == maxNumXYPlanes); assert(indices.size() == buffer.dim_mid() * numZSticks.size()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqXYData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(buffer.dim_mid(), gpu::GridSizeMedium)); assert(threadGrid.x > 0); assert(threadGrid.y > 0); launch_kernel(buffered_unpack_backward_kernel, threadGrid, threadBlock, 0, stream, numZSticks, indices, buffer, GPUArrayView2D(freqXYData.data(), freqXYData.dim_outer(), freqXYData.dim_mid() * freqXYData.dim_inner(), freqXYData.device_id())); } auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { buffered_unpack_backward_launch(stream, maxNumXYPlanes, numZSticks, indices, buffer, freqXYData); } auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { buffered_unpack_backward_launch(stream, maxNumXYPlanes, numZSticks, indices, buffer, freqXYData); } auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { buffered_unpack_backward_launch(stream, maxNumXYPlanes, numZSticks, indices, buffer, freqXYData); } // Unpacks z-sticks from buffer after MPI_Alltoall // Dimension of buffer are (numRanks, maxNumZSticks, maxNumXYPlanes) // Dimension of freqZData are (numLocalZSticks, dimZ) template __global__ static void buffered_unpack_forward_kernel(const GPUArrayConstView1D numXYPlanes, const GPUArrayConstView1D xyPlaneOffsets, const GPUArrayConstView3D buffer, GPUArrayView2D freqZData) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; for (int r = 0; r < numXYPlanes.size(); ++r) { if (xyPlaneIndex < numXYPlanes(r)) { const int xyOffset = xyPlaneOffsets(r); for (int zStickIndex = blockIdx.y; zStickIndex < freqZData.dim_outer(); zStickIndex += gridDim.y) { freqZData(zStickIndex, xyPlaneIndex + xyOffset) = ConvertComplex::apply(buffer(r, zStickIndex, xyPlaneIndex)); } } } } template static auto buffered_unpack_forward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D& buffer, GPUArrayView2D freqZData) -> void { assert(xyPlaneOffsets.size() == numXYPlanes.size()); assert(buffer.size() >= freqZData.size()); assert(buffer.dim_outer() == xyPlaneOffsets.size()); assert(buffer.dim_inner() == maxNumXYPlanes); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((maxNumXYPlanes + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); assert(threadGrid.x > 0); assert(threadGrid.y > 0); launch_kernel(buffered_unpack_forward_kernel, threadGrid, threadBlock, 0, stream, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } // Packs z-sticks into buffer for MPI_Alltoall // Dimension of buffer are (numRanks, maxNumZSticks, maxNumXYPlanes) // Dimension of freqXYData are (numLocalXYPlanes, dimY, dimX) template __global__ static void buffered_pack_forward_kernel( const GPUArrayConstView1D numZSticks, const GPUArrayConstView1D indices, const GPUArrayConstView2D freqXYDataFlat, GPUArrayView3D buffer) { // buffer.dim_mid() is equal to maxNumZSticks const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; if (xyPlaneIndex < freqXYDataFlat.dim_outer()) { for (int r = 0; r < numZSticks.size(); ++r) { const int numCurrentZSticks = numZSticks(r); for (int zStickIndex = blockIdx.y; zStickIndex < numCurrentZSticks; zStickIndex += gridDim.y) { const int currentIndex = indices(r * buffer.dim_mid() + zStickIndex); buffer(r, zStickIndex, xyPlaneIndex) = ConvertComplex::apply( freqXYDataFlat(xyPlaneIndex, currentIndex)); } } } } template static auto buffered_pack_forward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D& freqXYData, GPUArrayView3D buffer) -> void { assert(buffer.dim_outer() == numZSticks.size()); assert(buffer.dim_inner() == maxNumXYPlanes); assert(indices.size() == buffer.dim_mid() * numZSticks.size()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqXYData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(buffer.dim_mid(), gpu::GridSizeMedium)); assert(threadGrid.x > 0); assert(threadGrid.y > 0); launch_kernel(buffered_pack_forward_kernel, threadGrid, threadBlock, 0, stream, numZSticks, indices, GPUArrayConstView2D(freqXYData.data(), freqXYData.dim_outer(), freqXYData.dim_mid() * freqXYData.dim_inner(), freqXYData.device_id()), buffer); } auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void { buffered_pack_forward_launch(stream, maxNumXYPlanes, numZSticks, indices, freqXYData, buffer); } auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void { buffered_pack_forward_launch(stream, maxNumXYPlanes, numZSticks, indices, freqXYData, buffer); } auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void { buffered_pack_forward_launch(stream, maxNumXYPlanes, numZSticks, indices, freqXYData, buffer); } } // namespace spfft SpFFT-0.9.10/src/transpose/gpu_kernels/buffered_kernels.hpp000066400000000000000000000133101357247005600236730ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_BUFFERED_KERNELS_HPP #define SPFFT_BUFFERED_KERNELS_HPP #include #include "gpu_util/gpu_fft_api.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void; auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void; auto buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> buffer) -> void; auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto buffered_unpack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView3D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void; auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void; auto buffered_pack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView3D::type> buffer) -> void; } // namespace spfft #endif SpFFT-0.9.10/src/transpose/gpu_kernels/compact_buffered_kernels.cu000066400000000000000000000376421357247005600252370ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "gpu_util/complex_conversion.cuh" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_kernel_parameter.hpp" #include "gpu_util/gpu_runtime.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array_const_view.hpp" namespace spfft { template __global__ static void compact_buffered_pack_backward_kernel( const GPUArrayConstView1D numXYPlanes, const GPUArrayConstView1D xyPlaneOffsets, const GPUArrayConstView2D freqZData, GPUArrayView1D buffer) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; int bufferOffset = 0; for (int r = 0; r < numXYPlanes.size(); ++r) { const int numCurrentXYPlanes = numXYPlanes(r); if (xyPlaneIndex < numCurrentXYPlanes) { for (int zStickIndex = blockIdx.y; zStickIndex < freqZData.dim_outer(); zStickIndex += gridDim.y) { buffer(bufferOffset + zStickIndex * numCurrentXYPlanes + xyPlaneIndex) = ConvertComplex::apply( freqZData(zStickIndex, xyPlaneIndex + xyPlaneOffsets(r))); } } bufferOffset += numCurrentXYPlanes * freqZData.dim_outer(); } } template static auto compact_buffered_pack_backward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D& freqZData, GPUArrayView1D buffer) -> void { assert(xyPlaneOffsets.size() == numXYPlanes.size()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((maxNumXYPlanes + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(compact_buffered_pack_backward_kernel, threadGrid, threadBlock, 0, stream, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_backward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, freqZData, buffer); } template __global__ static void compact_buffered_unpack_backward_kernel( const int maxNumZSticks, const GPUArrayConstView1D numZSticks, const GPUArrayConstView1D indices, const GPUArrayConstView1D buffer, GPUArrayView2D freqXYData) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; int bufferOffset = 0; if (xyPlaneIndex < freqXYData.dim_outer()) { for (int r = 0; r < numZSticks.size(); ++r) { const int numCurrentZSticks = numZSticks(r); for (int zStickIndex = blockIdx.y; zStickIndex < numCurrentZSticks; zStickIndex += gridDim.y) { const int currentIndex = indices(r * maxNumZSticks + zStickIndex); freqXYData(xyPlaneIndex, currentIndex) = ConvertComplex::apply( buffer(bufferOffset + zStickIndex * freqXYData.dim_outer() + xyPlaneIndex)); } bufferOffset += numCurrentZSticks * freqXYData.dim_outer(); } } } template static auto compact_buffered_unpack_backward_launch(const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D& buffer, GPUArrayView3D freqXYData) -> void { const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqXYData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(maxNumZSticks, gpu::GridSizeMedium)); launch_kernel(compact_buffered_unpack_backward_kernel, threadGrid, threadBlock, 0, stream, maxNumZSticks, numZSticks, indices, buffer, GPUArrayView2D(freqXYData.data(), freqXYData.dim_outer(), freqXYData.dim_mid() * freqXYData.dim_inner(), freqXYData.device_id())); } auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { compact_buffered_unpack_backward_launch(stream, maxNumZSticks, numZSticks, indices, buffer, freqXYData); } auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { compact_buffered_unpack_backward_launch(stream, maxNumZSticks, numZSticks, indices, buffer, freqXYData); } auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void { compact_buffered_unpack_backward_launch(stream, maxNumZSticks, numZSticks, indices, buffer, freqXYData); } template __global__ static void compact_buffered_unpack_forward_kernel( const GPUArrayConstView1D numXYPlanes, const GPUArrayConstView1D xyPlaneOffsets, const GPUArrayConstView1D buffer, GPUArrayView2D freqZData) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; int bufferOffset = 0; for (int r = 0; r < numXYPlanes.size(); ++r) { const int numCurrentXYPlanes = numXYPlanes(r); if (xyPlaneIndex < numCurrentXYPlanes) { for (int zStickIndex = blockIdx.y; zStickIndex < freqZData.dim_outer(); zStickIndex += gridDim.y) { freqZData(zStickIndex, xyPlaneIndex + xyPlaneOffsets(r)) = ConvertComplex::apply( buffer(bufferOffset + zStickIndex * numCurrentXYPlanes + xyPlaneIndex)); } } bufferOffset += numCurrentXYPlanes * freqZData.dim_outer(); } } template static auto compact_buffered_unpack_forward_launch(const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D& buffer, GPUArrayView2D freqZData) -> void { assert(xyPlaneOffsets.size() == numXYPlanes.size()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((maxNumXYPlanes + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(compact_buffered_unpack_forward_kernel, threadGrid, threadBlock, 0, stream, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { compact_buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { compact_buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void { compact_buffered_unpack_forward_launch(stream, maxNumXYPlanes, numXYPlanes, xyPlaneOffsets, buffer, freqZData); } template __global__ static void compact_buffered_pack_forward_kernel( const int maxNumZSticks, const GPUArrayConstView1D numZSticks, const GPUArrayConstView1D indices, const GPUArrayConstView2D freqXYData, GPUArrayView1D buffer) { const int xyPlaneIndex = threadIdx.x + blockIdx.x * blockDim.x; int bufferOffset = 0; if (xyPlaneIndex < freqXYData.dim_outer()) { for (int r = 0; r < numZSticks.size(); ++r) { const int numCurrentZSticks = numZSticks(r); for (int zStickIndex = blockIdx.y; zStickIndex < numCurrentZSticks; zStickIndex += gridDim.y) { const int currentIndex = indices(r * maxNumZSticks + zStickIndex); buffer(bufferOffset + zStickIndex * freqXYData.dim_outer() + xyPlaneIndex) = ConvertComplex::apply(freqXYData(xyPlaneIndex, currentIndex)); } bufferOffset += numCurrentZSticks * freqXYData.dim_outer(); } } } template static auto compact_buffered_pack_forward_launch(const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D& freqXYData, GPUArrayView1D buffer) -> void { const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqXYData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(maxNumZSticks, gpu::GridSizeMedium)); launch_kernel(compact_buffered_pack_forward_kernel, threadGrid, threadBlock, 0, stream, maxNumZSticks, numZSticks, indices, GPUArrayConstView2D(freqXYData.data(), freqXYData.dim_outer(), freqXYData.dim_mid() * freqXYData.dim_inner(), freqXYData.device_id()), buffer); } auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_forward_launch(stream, maxNumZSticks, numZSticks, indices, freqXYData, buffer); } auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_forward_launch(stream, maxNumZSticks, numZSticks, indices, freqXYData, buffer); } auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void { compact_buffered_pack_forward_launch(stream, maxNumZSticks, numZSticks, indices, freqXYData, buffer); } } // namespace spfft SpFFT-0.9.10/src/transpose/gpu_kernels/compact_buffered_kernels.hpp000066400000000000000000000134541357247005600254120ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_COMPACT_BUFFERED_KERNELS_HPP #define SPFFT_COMPACT_BUFFERED_KERNELS_HPP #include #include "gpu_util/gpu_fft_api.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void; auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void; auto compact_buffered_pack_backward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView2D::type>& freqZData, GPUArrayView1D::type> buffer) -> void; auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto compact_buffered_unpack_backward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView1D::type>& buffer, GPUArrayView3D::type> freqXYData) -> void; auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto compact_buffered_unpack_forward( const gpu::StreamType stream, const int maxNumXYPlanes, const GPUArrayView1D numXYPlanes, const GPUArrayView1D& xyPlaneOffsets, const GPUArrayView1D::type>& buffer, GPUArrayView2D::type> freqZData) -> void; auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void; auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void; auto compact_buffered_pack_forward( const gpu::StreamType stream, const int maxNumZSticks, const GPUArrayView1D& numZSticks, const GPUArrayView1D& indices, const GPUArrayView3D::type>& freqXYData, GPUArrayView1D::type> buffer) -> void; } // namespace spfft #endif SpFFT-0.9.10/src/transpose/gpu_kernels/local_transpose_kernels.cu000066400000000000000000000243751357247005600251360ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_kernel_parameter.hpp" #include "gpu_util/gpu_runtime.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array_const_view.hpp" #include "spfft/config.h" namespace spfft { // ------------------ // Backward // ------------------ #ifdef SPFFT_CUDA // kernel optimized for NVIDIA // Places data from z-sticks into a full 3d grid template __global__ static void transpose_backward_kernel(const GPUArrayConstView1D indices, const GPUArrayConstView2D freqZData, GPUArrayView2D spaceDomainFlat) { // const int z = threadIdx.x + blockIdx.x * blockDim.x; const int stickIndex = threadIdx.x + blockIdx.x * blockDim.x; if (stickIndex < indices.size()) { const auto stickXYIndex = indices(stickIndex); for (int z = blockIdx.y; z < freqZData.dim_inner(); z += gridDim.y) { spaceDomainFlat(z, stickXYIndex) = freqZData(stickIndex, z); } } } auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_inner(), gpu::GridSizeMedium)); launch_kernel(transpose_backward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, freqZData, GPUArrayView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id())); } auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_outer() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_inner(), gpu::GridSizeMedium)); launch_kernel(transpose_backward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, freqZData, GPUArrayView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id())); } #else // kernel optimized for AMD // (ideal memory access pattern is different) template __global__ static void transpose_backward_kernel(const GPUArrayConstView1D indices, const GPUArrayConstView2D freqZData, GPUArrayView2D spaceDomainFlat) { const int z = threadIdx.x + blockIdx.x * blockDim.x; if (z < freqZData.dim_inner()) { for (int stickIndex = blockIdx.y; stickIndex < indices.size(); stickIndex += gridDim.y) { const auto stickXYIndex = indices(stickIndex); spaceDomainFlat(z, stickXYIndex) = freqZData(stickIndex, z); } } } auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_inner() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(transpose_backward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, freqZData, GPUArrayView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id())); } auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_inner() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(transpose_backward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, freqZData, GPUArrayView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id())); } #endif // ------------------ // Forward // ------------------ template __global__ static void transpose_forward_kernel(const GPUArrayConstView1D indices, const GPUArrayConstView2D spaceDomainFlat, GPUArrayView2D freqZData) { const int z = threadIdx.x + blockIdx.x * blockDim.x; if (z < freqZData.dim_inner()) { for (int stickIndex = blockIdx.y; stickIndex < indices.size(); stickIndex += gridDim.y) { const auto stickXYIndex = indices(stickIndex); freqZData(stickIndex, z) = spaceDomainFlat(z, stickXYIndex); } } } auto local_transpose_forward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView3D::type>& spaceDomain, GPUArrayView2D::type> freqZData) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_inner() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(transpose_forward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, GPUArrayConstView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id()), freqZData); } auto local_transpose_forward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView3D::type>& spaceDomain, GPUArrayView2D::type> freqZData) -> void { assert(indices.size() == freqZData.dim_outer()); assert(indices.size() <= spaceDomain.dim_inner() * spaceDomain.dim_mid()); assert(spaceDomain.dim_outer() == freqZData.dim_inner()); const dim3 threadBlock(gpu::BlockSizeSmall); const dim3 threadGrid((freqZData.dim_inner() + threadBlock.x - 1) / threadBlock.x, std::min(freqZData.dim_outer(), gpu::GridSizeMedium)); launch_kernel(transpose_forward_kernel::type>, threadGrid, threadBlock, 0, stream, indices, GPUArrayConstView2D::type>( spaceDomain.data(), spaceDomain.dim_outer(), spaceDomain.dim_mid() * spaceDomain.dim_inner(), spaceDomain.device_id()), freqZData); } } // namespace spfft SpFFT-0.9.10/src/transpose/gpu_kernels/local_transpose_kernels.hpp000066400000000000000000000055111357247005600253050ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include "gpu_util/gpu_fft_api.hpp" #include "memory/gpu_array_view.hpp" namespace spfft { // ------------------ // Backward // ------------------ auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void; auto local_transpose_backward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView2D::type>& freqZData, GPUArrayView3D::type> spaceDomain) -> void; // ------------------ // Forward // ------------------ auto local_transpose_forward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView3D::type>& spaceDomain, GPUArrayView2D::type> freqZData) -> void; auto local_transpose_forward( const gpu::StreamType stream, const GPUArrayView1D indices, const GPUArrayView3D::type>& spaceDomain, GPUArrayView2D::type> freqZData) -> void; } // namespace spfft SpFFT-0.9.10/src/transpose/transpose.hpp000066400000000000000000000050371357247005600200750ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_HPP #define SPFFT_TRANSPOSE_HPP #include "spfft/config.h" #include "util/common_types.hpp" namespace spfft { class Transpose { public: virtual auto pack_forward() -> void {} virtual auto exchange_forward_start(const bool nonBlockingExchange) -> void = 0; virtual auto exchange_forward_finalize() -> void {} virtual auto unpack_forward() -> void {} inline auto forward() -> void { this->pack_forward(); this->exchange_forward_start(false); this->exchange_forward_finalize(); this->unpack_forward(); } virtual auto pack_backward() -> void {} virtual auto exchange_backward_start(const bool nonBlockingExchange) -> void = 0; virtual auto exchange_backward_finalize() -> void {} virtual auto unpack_backward() -> void {} inline auto backward() -> void { this->pack_backward(); this->exchange_backward_start(false); this->exchange_backward_finalize(); this->unpack_backward(); } virtual ~Transpose() = default; }; } // namespace spfft #endif SpFFT-0.9.10/src/transpose/transpose_gpu.hpp000066400000000000000000000115631357247005600207510ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_GPU_HPP #define SPFFT_TRANSPOSE_GPU_HPP #include #include #include #include #include #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/array_view_utility.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "transpose/gpu_kernels/local_transpose_kernels.hpp" namespace spfft { // Transpose Z sticks, such that data is represented by xy planes, where the y-dimension is // continous and vice versa template class TransposeGPU : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = typename gpu::fft::ComplexType::type; public: TransposeGPU(const std::shared_ptr& param, GPUStreamHandle stream, GPUArrayView3D spaceDomainData, GPUArrayView2D freqDomainData) : stream_(std::move(stream)), spaceDomainData_(spaceDomainData), freqDomainData_(freqDomainData), indices_(param->num_z_sticks(0)) { // single node only checks assert(spaceDomainData.dim_outer() == freqDomainData.dim_inner()); // check data dimensions and parameters assert(param->dim_x_freq() == spaceDomainData.dim_inner()); assert(param->dim_y() == spaceDomainData.dim_mid()); assert(param->dim_z() == spaceDomainData.dim_outer()); assert(param->dim_z() == freqDomainData.dim_inner()); assert(param->num_z_sticks(0) == freqDomainData.dim_outer()); // data must be disjoint assert(disjoint(spaceDomainData, freqDomainData)); // copy xy indices const auto zStickXYIndices = param->z_stick_xy_indices(0); std::vector transposedIndices; transposedIndices.reserve(zStickXYIndices.size()); for (const auto& index : zStickXYIndices) { const int x = index / param->dim_y(); const int y = index - x * param->dim_y(); transposedIndices.emplace_back(y * param->dim_x_freq() + x); } copy_to_gpu(transposedIndices, indices_); } auto exchange_backward_start(const bool) -> void override { gpu::check_status(gpu::memset_async( static_cast(spaceDomainData_.data()), 0, spaceDomainData_.size() * sizeof(typename decltype(spaceDomainData_)::ValueType), stream_.get())); if (freqDomainData_.size() > 0 && spaceDomainData_.size() > 0) { local_transpose_backward(stream_.get(), create_1d_view(indices_, 0, indices_.size()), freqDomainData_, spaceDomainData_); } } auto unpack_backward() -> void override {} auto exchange_forward_start(const bool) -> void override { if (freqDomainData_.size() > 0 && spaceDomainData_.size() > 0) { local_transpose_forward(stream_.get(), create_1d_view(indices_, 0, indices_.size()), spaceDomainData_, freqDomainData_); } } auto unpack_forward() -> void override {} private: GPUStreamHandle stream_; GPUArrayView3D spaceDomainData_; GPUArrayView2D freqDomainData_; GPUArray indices_; }; } // namespace spfft #endif SpFFT-0.9.10/src/transpose/transpose_host.hpp000066400000000000000000000163011357247005600211260ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_HOST_HPP #define SPFFT_TRANSPOSE_HOST_HPP #include #include #include #include #include #include #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" namespace spfft { // Transpose Z sticks, such that data is represented by xy planes, where the y-dimension is // continous and vice versa template class TransposeHost : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; public: TransposeHost(const std::shared_ptr& param, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData) : spaceDomainData_(spaceDomainData), freqDomainData_(freqDomainData), param_(param) { // single rank only checks assert(spaceDomainData.dim_outer() == freqDomainData.dim_inner()); // check data dimensions and parameters assert(param->dim_x_freq() == spaceDomainData.dim_mid()); assert(param->dim_y() == spaceDomainData.dim_inner()); assert(param->dim_z() == spaceDomainData.dim_outer()); assert(param->dim_z() == freqDomainData.dim_inner()); assert(param->num_z_sticks(0) == freqDomainData.dim_outer()); // data must be disjoint assert(disjoint(spaceDomainData, freqDomainData)); } auto exchange_backward_start(const bool) -> void override {} auto unpack_backward() -> void override { SPFFT_OMP_PRAGMA("omp for schedule(static)") // implicit barrier for (SizeType z = 0; z < spaceDomainData_.dim_outer(); ++z) { std::memset(static_cast(&spaceDomainData_(z, 0, 0)), 0, sizeof(typename decltype(spaceDomainData_)::ValueType) * spaceDomainData_.dim_inner() * spaceDomainData_.dim_mid()); } const SizeType unrolledLoopEnd = freqDomainData_.dim_outer() < 4 ? 0 : freqDomainData_.dim_outer() - 3; auto stickIndicesView = param_->z_stick_xy_indices(0); auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // unrolled loop SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { const SizeType xyIndex1 = stickIndicesView(zStickIndex); const SizeType xyIndex2 = stickIndicesView(zStickIndex + 1); const SizeType xyIndex3 = stickIndicesView(zStickIndex + 2); const SizeType xyIndex4 = stickIndicesView(zStickIndex + 3); for (SizeType zIndex = 0; zIndex < freqDomainData_.dim_inner(); ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex1) = freqDomainData_(zStickIndex, zIndex); spaceDomainDataFlat(zIndex, xyIndex2) = freqDomainData_(zStickIndex + 1, zIndex); spaceDomainDataFlat(zIndex, xyIndex3) = freqDomainData_(zStickIndex + 2, zIndex); spaceDomainDataFlat(zIndex, xyIndex4) = freqDomainData_(zStickIndex + 3, zIndex); } } // transpose remaining elements SPFFT_OMP_PRAGMA("omp for schedule(static)") // keep barrier for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < freqDomainData_.dim_outer(); zStickIndex += 1) { const SizeType xyIndex = stickIndicesView(zStickIndex); for (SizeType zIndex = 0; zIndex < freqDomainData_.dim_inner(); ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex) = freqDomainData_(zStickIndex, zIndex); } } } auto exchange_forward_start(const bool) -> void override {} auto unpack_forward() -> void override { const SizeType unrolledLoopEnd = freqDomainData_.dim_outer() < 4 ? 0 : freqDomainData_.dim_outer() - 3; auto stickIndicesView = param_->z_stick_xy_indices(0); auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // unrolled loop SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { const SizeType xyIndex1 = stickIndicesView(zStickIndex); const SizeType xyIndex2 = stickIndicesView(zStickIndex + 1); const SizeType xyIndex3 = stickIndicesView(zStickIndex + 2); const SizeType xyIndex4 = stickIndicesView(zStickIndex + 3); for (SizeType zIndex = 0; zIndex < freqDomainData_.dim_inner(); ++zIndex) { freqDomainData_(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex1); freqDomainData_(zStickIndex + 1, zIndex) = spaceDomainDataFlat(zIndex, xyIndex2); freqDomainData_(zStickIndex + 2, zIndex) = spaceDomainDataFlat(zIndex, xyIndex3); freqDomainData_(zStickIndex + 3, zIndex) = spaceDomainDataFlat(zIndex, xyIndex4); } } // transpose remaining elements SPFFT_OMP_PRAGMA("omp for schedule(static)") // keep barrier for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < freqDomainData_.dim_outer(); zStickIndex += 1) { const SizeType xyIndex = stickIndicesView(zStickIndex); for (SizeType zIndex = 0; zIndex < freqDomainData_.dim_inner(); ++zIndex) { freqDomainData_(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex); } } } private: HostArrayView3D spaceDomainData_; HostArrayView2D freqDomainData_; std::shared_ptr param_; }; } // namespace spfft #endif SpFFT-0.9.10/src/transpose/transpose_mpi_buffered_gpu.cpp000066400000000000000000000306641357247005600234560ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #if defined(SPFFT_MPI) && (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_transfer.hpp" #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/gpu_kernels/buffered_kernels.hpp" #include "transpose/transpose_mpi_buffered_gpu.hpp" namespace spfft { template TransposeMPIBufferedGPU::TransposeMPIBufferedGPU( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView1D spaceDomainBufferHost, GPUArrayView3D spaceDomainDataGPU, GPUArrayView1D spaceDomainBufferGPU, GPUStreamHandle spaceDomainStream, HostArrayView1D freqDomainBufferHost, GPUArrayView2D freqDomainDataGPU, GPUArrayView1D freqDomainBufferGPU, GPUStreamHandle freqDomainStream) : param_(param), comm_(std::move(comm)), spaceDomainBufferHost_(create_new_type_1d_view( spaceDomainBufferHost, comm_.size() * param_->max_num_xy_planes() * param_->max_num_z_sticks())), freqDomainBufferHost_(create_new_type_1d_view( freqDomainBufferHost, comm_.size() * param_->max_num_xy_planes() * param_->max_num_z_sticks())), spaceDomainDataGPU_(spaceDomainDataGPU), freqDomainDataGPU_(freqDomainDataGPU), spaceDomainBufferGPU_(create_new_type_3d_view( spaceDomainBufferGPU, comm_.size(), param_->max_num_z_sticks(), param_->max_num_xy_planes())), freqDomainBufferGPU_(create_new_type_3d_view( freqDomainBufferGPU, comm_.size(), param_->max_num_z_sticks(), param_->max_num_xy_planes())), spaceDomainStream_(std::move(spaceDomainStream)), freqDomainStream_(std::move(freqDomainStream)) { assert(param_->dim_y() == spaceDomainDataGPU.dim_mid()); assert(param_->dim_x_freq() == spaceDomainDataGPU.dim_inner()); assert(param_->num_xy_planes(comm_.rank()) == spaceDomainDataGPU.dim_outer()); assert(param_->dim_z() == freqDomainDataGPU.dim_inner()); assert(param_->num_z_sticks(comm_.rank()) == freqDomainDataGPU.dim_outer()); assert(spaceDomainBufferGPU.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); assert(freqDomainBufferGPU.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); assert(spaceDomainBufferHost.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); assert(freqDomainBufferHost.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); // assert(disjoint(spaceDomainDataGPU, freqDomainDataGPU)); assert(disjoint(spaceDomainDataGPU, spaceDomainBufferGPU)); assert(disjoint(freqDomainDataGPU, freqDomainBufferGPU)); assert(disjoint(spaceDomainBufferHost, freqDomainBufferHost)); #ifdef SPFFT_GPU_DIRECT assert(disjoint(spaceDomainBufferGPU, freqDomainBufferGPU)); #endif // create underlying type mpiTypeHandle_ = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); // copy relevant parameters std::vector numZSticksHost(comm_.size()); std::vector numXYPlanesHost(comm_.size()); std::vector xyPlaneOffsetsHost(comm_.size()); std::vector indicesHost(comm_.size() * param_->max_num_z_sticks()); for (SizeType r = 0; r < comm_.size(); ++r) { numZSticksHost[r] = static_cast(param_->num_z_sticks(r)); numXYPlanesHost[r] = static_cast(param_->num_xy_planes(r)); xyPlaneOffsetsHost[r] = static_cast(param_->xy_plane_offset(r)); const auto zStickXYIndices = param_->z_stick_xy_indices(r); for (SizeType i = 0; i < zStickXYIndices.size(); ++i) { // transpose stick index const int xyIndex = zStickXYIndices(i); const int x = xyIndex / param_->dim_y(); const int y = xyIndex - x * param_->dim_y(); indicesHost[r * param_->max_num_z_sticks() + i] = y * param_->dim_x_freq() + x; } } numZSticksGPU_ = GPUArray(numZSticksHost.size()); numXYPlanesGPU_ = GPUArray(numXYPlanesHost.size()); xyPlaneOffsetsGPU_ = GPUArray(xyPlaneOffsetsHost.size()); indicesGPU_ = GPUArray(indicesHost.size()); copy_to_gpu(numZSticksHost, numZSticksGPU_); copy_to_gpu(numXYPlanesHost, numXYPlanesGPU_); copy_to_gpu(xyPlaneOffsetsHost, xyPlaneOffsetsGPU_); copy_to_gpu(indicesHost, indicesGPU_); } template auto TransposeMPIBufferedGPU::pack_backward() -> void { if (freqDomainDataGPU_.size() > 0 && freqDomainBufferGPU_.size() > 0) { buffered_pack_backward(freqDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numXYPlanesGPU_, 0, numXYPlanesGPU_.size()), create_1d_view(xyPlaneOffsetsGPU_, 0, xyPlaneOffsetsGPU_.size()), freqDomainDataGPU_, freqDomainBufferGPU_); #ifndef SPFFT_GPU_DIRECT copy_from_gpu_async(freqDomainStream_, freqDomainBufferGPU_, freqDomainBufferHost_); #endif } } template auto TransposeMPIBufferedGPU::unpack_backward() -> void { if (spaceDomainDataGPU_.size() > 0) { gpu::check_status(gpu::memset_async( static_cast(spaceDomainDataGPU_.data()), 0, spaceDomainDataGPU_.size() * sizeof(typename decltype(spaceDomainDataGPU_)::ValueType), spaceDomainStream_.get())); if (spaceDomainBufferGPU_.size() > 0) { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(spaceDomainStream_, spaceDomainBufferHost_, spaceDomainBufferGPU_); #endif buffered_unpack_backward(spaceDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numZSticksGPU_, 0, numZSticksGPU_.size()), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), spaceDomainBufferGPU_, spaceDomainDataGPU_); } } } template auto TransposeMPIBufferedGPU::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only master thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(freqDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = freqDomainBufferGPU_.data(); auto recvBufferPtr = spaceDomainBufferGPU_.data(); #else auto sendBufferPtr = freqDomainBufferHost_.data(); auto recvBufferPtr = spaceDomainBufferHost_.data(); #endif if (nonBlockingExchange) { // start non-blocking exchange mpi_check_status( MPI_Ialltoall(sendBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), recvBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { // blocking exchange mpi_check_status(MPI_Alltoall(sendBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), recvBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPIBufferedGPU::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPIBufferedGPU::pack_forward() -> void { if (spaceDomainDataGPU_.size() > 0 && spaceDomainBufferGPU_.size() > 0) { buffered_pack_forward(spaceDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numZSticksGPU_, 0, numZSticksGPU_.size()), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), spaceDomainDataGPU_, spaceDomainBufferGPU_); #ifndef SPFFT_GPU_DIRECT copy_from_gpu_async(spaceDomainStream_, spaceDomainBufferGPU_, spaceDomainBufferHost_); #endif } } template auto TransposeMPIBufferedGPU::unpack_forward() -> void { if (freqDomainDataGPU_.size() > 0 && freqDomainBufferGPU_.size() > 0) { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(freqDomainStream_, freqDomainBufferHost_, freqDomainBufferGPU_); #endif buffered_unpack_forward(freqDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numXYPlanesGPU_, 0, numXYPlanesGPU_.size()), create_1d_view(xyPlaneOffsetsGPU_, 0, xyPlaneOffsetsGPU_.size()), freqDomainBufferGPU_, freqDomainDataGPU_); } } template auto TransposeMPIBufferedGPU::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only master thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(spaceDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = spaceDomainBufferGPU_.data(); auto recvBufferPtr = freqDomainBufferGPU_.data(); #else auto sendBufferPtr = spaceDomainBufferHost_.data(); auto recvBufferPtr = freqDomainBufferHost_.data(); #endif if (nonBlockingExchange) { // start non-blocking exchange mpi_check_status( MPI_Ialltoall(sendBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), recvBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { // blocking exchange mpi_check_status(MPI_Alltoall(sendBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), recvBufferPtr, param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPIBufferedGPU::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPIBufferedGPU; #endif template class TransposeMPIBufferedGPU; template class TransposeMPIBufferedGPU; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_buffered_gpu.hpp000066400000000000000000000116301357247005600234530ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_BUFFERED_GPU_HPP #define SPFFT_TRANSPOSE_MPI_BUFFERED_GPU_HPP #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #if defined(SPFFT_MPI) && (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPIBufferedGPU : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; using ComplexExchangeType = std::complex; using ComplexGPUType = typename gpu::fft::ComplexType::type; using ComplexExchangeGPUType = typename gpu::fft::ComplexType::type; public: // spaceDomainDataGPU and freqDomainDataGPU must NOT overlap // spaceDomainDataGPU and spaceDomainBufferGPU must NOT overlap // freqDomainDataGPU and freqDomainBufferGPU must NOT overlap // spaceDomainBufferGPU and freqDomainBufferGPU must NOT overlap // spaceDomainBufferHost and freqDomainBufferHost must NOT overlap // // spaceDomainBufferGPU and freqDomainDataGPU MAY overlap // freqDomainBufferGPU and spaceDomainDataGPU MAY overlap TransposeMPIBufferedGPU(const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView1D spaceDomainBufferHost, GPUArrayView3D spaceDomainDataGPU, GPUArrayView1D spaceDomainBufferGPU, GPUStreamHandle spaceDomainStream, HostArrayView1D freqDomainBufferHost, GPUArrayView2D freqDomainDataGPU, GPUArrayView1D freqDomainBufferGPU, GPUStreamHandle freqDomainStream); auto pack_backward() -> void override; auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto unpack_backward() -> void override; auto pack_forward() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; auto unpack_forward() -> void override; private: std::shared_ptr param_; MPIDatatypeHandle mpiTypeHandle_; MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; HostArrayView1D spaceDomainBufferHost_; HostArrayView1D freqDomainBufferHost_; GPUArrayView3D spaceDomainDataGPU_; GPUArrayView2D freqDomainDataGPU_; GPUArrayView3D spaceDomainBufferGPU_; GPUArrayView3D freqDomainBufferGPU_; GPUStreamHandle spaceDomainStream_; GPUStreamHandle freqDomainStream_; GPUArray numZSticksGPU_; GPUArray numXYPlanesGPU_; GPUArray xyPlaneOffsetsGPU_; GPUArray indicesGPU_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/transpose/transpose_mpi_buffered_host.cpp000066400000000000000000000317771357247005600236460ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/transpose_mpi_buffered_host.hpp" namespace spfft { template TransposeMPIBufferedHost::TransposeMPIBufferedHost( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData, HostArrayView1D spaceDomainBuffer, HostArrayView1D freqDomainBuffer) : param_(param), comm_(std::move(comm)), spaceDomainData_(spaceDomainData), freqDomainData_(freqDomainData), spaceDomainBuffer_(create_new_type_1d_view(spaceDomainBuffer, spaceDomainBuffer.size())), freqDomainBuffer_( create_new_type_1d_view(freqDomainBuffer, freqDomainBuffer.size())) { // assert(param_->dim_x_freq() == spaceDomainData.dim_mid()); assert(param_->dim_y() == spaceDomainData.dim_inner()); assert(param_->num_xy_planes(comm_.rank()) == spaceDomainData.dim_outer()); assert(param_->dim_z() == freqDomainData.dim_inner()); assert(param_->num_z_sticks(comm_.rank()) == freqDomainData.dim_outer()); assert(spaceDomainBuffer.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); assert(freqDomainBuffer.size() >= param_->max_num_xy_planes() * param_->max_num_z_sticks() * comm_.size()); assert(disjoint(spaceDomainData, freqDomainData)); assert(disjoint(spaceDomainData, spaceDomainBuffer)); assert(disjoint(freqDomainData, freqDomainBuffer)); assert(disjoint(spaceDomainBuffer, freqDomainBuffer)); // create underlying type mpiTypeHandle_ = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); } template auto TransposeMPIBufferedHost::pack_backward() -> void { auto freqDomainBuffer3d = create_3d_view(freqDomainBuffer_, 0, comm_.size(), param_->max_num_z_sticks(), param_->max_num_xy_planes()); // transpose locally from (numLocalZSticks, dimZ) to (dimZ, numLocalZSticks) with spacing // between ranks for (SizeType r = 0; r < static_cast(comm_.size()); ++r) { const auto xyPlaneOffset = param_->xy_plane_offset(r); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < freqDomainData_.dim_outer(); ++zStickIndex) { for (SizeType xyPlaneIndex = 0; xyPlaneIndex < param_->num_xy_planes(r); ++xyPlaneIndex) { freqDomainBuffer3d(r, zStickIndex, xyPlaneIndex) = freqDomainData_(zStickIndex, xyPlaneIndex + xyPlaneOffset); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPIBufferedHost::unpack_backward() -> void { // zero target data location (not all values are overwritten upon unpacking) SPFFT_OMP_PRAGMA("omp for schedule(static)") // implicit barrier for (SizeType z = 0; z < spaceDomainData_.dim_outer(); ++z) { std::memset(static_cast(&spaceDomainData_(z, 0, 0)), 0, sizeof(typename decltype(spaceDomainData_)::ValueType) * spaceDomainData_.dim_inner() * spaceDomainData_.dim_mid()); } auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // unpack from (numZSticksTotal, numLocalXYPlanes) to (numLocalXYPlanes, dimX, dimY) const auto numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { const auto zStickXYIndices = param_->z_stick_xy_indices(r); // take care with unsigned type const SizeType unrolledLoopEnd = zStickXYIndices.size() < 4 ? 0 : zStickXYIndices.size() - 3; auto spaceDomainBuffer2d = create_2d_view( spaceDomainBuffer_, r * param_->max_num_xy_planes() * param_->max_num_z_sticks(), param_->max_num_z_sticks(), param_->max_num_xy_planes()); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { // manual loop unrolling for better performance const SizeType xyIndex1 = zStickXYIndices(zStickIndex); const SizeType xyIndex2 = zStickXYIndices(zStickIndex + 1); const SizeType xyIndex3 = zStickXYIndices(zStickIndex + 2); const SizeType xyIndex4 = zStickXYIndices(zStickIndex + 3); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex1) = spaceDomainBuffer2d(zStickIndex, zIndex); spaceDomainDataFlat(zIndex, xyIndex2) = spaceDomainBuffer2d(zStickIndex + 1, zIndex); spaceDomainDataFlat(zIndex, xyIndex3) = spaceDomainBuffer2d(zStickIndex + 2, zIndex); spaceDomainDataFlat(zIndex, xyIndex4) = spaceDomainBuffer2d(zStickIndex + 3, zIndex); } } SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < zStickXYIndices.size(); zStickIndex += 1) { const SizeType xyIndex = zStickXYIndices(zStickIndex); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex) = spaceDomainBuffer2d(zStickIndex, zIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPIBufferedHost::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only master thread must be allowed to enter // exchange data if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoall( freqDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), spaceDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoall(freqDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), spaceDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPIBufferedHost::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPIBufferedHost::pack_forward() -> void { auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // pack from (numLocalXYPlanes, dimX, dimY) to (numZSticksTotal, numLocalXYPlanes) const auto numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { const auto zStickXYIndices = param_->z_stick_xy_indices(r); // take care with unsigned type const SizeType unrolledLoopEnd = zStickXYIndices.size() < 4 ? 0 : zStickXYIndices.size() - 3; auto spaceDomainBuffer2d = create_2d_view( spaceDomainBuffer_, r * param_->max_num_xy_planes() * param_->max_num_z_sticks(), param_->max_num_z_sticks(), param_->max_num_xy_planes()); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { // manual loop unrolling for better performance const SizeType xyIndex1 = zStickXYIndices(zStickIndex); const SizeType xyIndex2 = zStickXYIndices(zStickIndex + 1); const SizeType xyIndex3 = zStickXYIndices(zStickIndex + 2); const SizeType xyIndex4 = zStickXYIndices(zStickIndex + 3); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainBuffer2d(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex1); spaceDomainBuffer2d(zStickIndex + 1, zIndex) = spaceDomainDataFlat(zIndex, xyIndex2); spaceDomainBuffer2d(zStickIndex + 2, zIndex) = spaceDomainDataFlat(zIndex, xyIndex3); spaceDomainBuffer2d(zStickIndex + 3, zIndex) = spaceDomainDataFlat(zIndex, xyIndex4); } } SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < zStickXYIndices.size(); zStickIndex += 1) { const SizeType xyIndex = zStickXYIndices(zStickIndex); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainBuffer2d(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPIBufferedHost::unpack_forward() -> void { auto freqDomainBuffer3d = create_3d_view(freqDomainBuffer_, 0, comm_.size(), param_->max_num_z_sticks(), param_->max_num_xy_planes()); for (SizeType r = 0; r < static_cast(comm_.size()); ++r) { const auto xyPlaneOffset = param_->xy_plane_offset(r); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < freqDomainData_.dim_outer(); ++zStickIndex) { for (SizeType xyPlaneIndex = 0; xyPlaneIndex < param_->num_xy_planes(r); ++xyPlaneIndex) { freqDomainData_(zStickIndex, xyPlaneIndex + xyPlaneOffset) = freqDomainBuffer3d(r, zStickIndex, xyPlaneIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPIBufferedHost::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only master thread must be allowed to enter // exchange data if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoall( spaceDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), freqDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoall(spaceDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), freqDomainBuffer_.data(), param_->max_num_z_sticks() * param_->max_num_xy_planes(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPIBufferedHost::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPIBufferedHost; #endif template class TransposeMPIBufferedHost; template class TransposeMPIBufferedHost; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_buffered_host.hpp000066400000000000000000000075331357247005600236440ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_BUFFERED_HOST_HPP #define SPFFT_TRANSPOSE_MPI_BUFFERED_HOST_HPP #include #include #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPIBufferedHost : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; using ComplexExchangeType = std::complex; public: // spaceDomainData and freqDomainData must NOT overlap // spaceDomainData and spaceDomainBuffer must NOT overlap // freqDomainData and freqDomainBuffer must NOT overlap // spaceDomainBuffer and freqDomainBuffer must NOT overlap // // spaceDomainBuffer and freqDomainData MAY overlap // freqDomainBuffer and spaceDomainData MAY overlap TransposeMPIBufferedHost(const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData, HostArrayView1D spaceDomainBuffer, HostArrayView1D freqDomainBuffer); auto pack_backward() -> void override; auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto unpack_backward() -> void override; auto pack_forward() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; auto unpack_forward() -> void override; private: std::shared_ptr param_; MPIDatatypeHandle mpiTypeHandle_; MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; HostArrayView3D spaceDomainData_; HostArrayView2D freqDomainData_; HostArrayView1D spaceDomainBuffer_; HostArrayView1D freqDomainBuffer_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/transpose/transpose_mpi_compact_buffered_gpu.cpp000066400000000000000000000326021357247005600251560ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "spfft/config.h" #if defined(SPFFT_MPI) && (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) #include #include #include #include #include #include #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_transfer.hpp" #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/gpu_kernels/compact_buffered_kernels.hpp" #include "transpose/transpose_mpi_compact_buffered_gpu.hpp" namespace spfft { template TransposeMPICompactBufferedGPU::TransposeMPICompactBufferedGPU( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView1D spaceDomainBufferHost, GPUArrayView3D spaceDomainDataGPU, GPUArrayView1D spaceDomainBufferGPU, GPUStreamHandle spaceDomainStream, HostArrayView1D freqDomainBufferHost, GPUArrayView2D freqDomainDataGPU, GPUArrayView1D freqDomainBufferGPU, GPUStreamHandle freqDomainStream) : param_(param), comm_(std::move(comm)), spaceDomainBufferHost_(create_new_type_1d_view( spaceDomainBufferHost, param_->num_xy_planes(comm_.rank()) * param_->total_num_z_sticks())), freqDomainBufferHost_(create_new_type_1d_view( freqDomainBufferHost, param_->total_num_xy_planes() * param_->num_z_sticks(comm_.rank()))), spaceDomainDataGPU_(spaceDomainDataGPU), freqDomainDataGPU_(freqDomainDataGPU), spaceDomainBufferGPU_(create_new_type_1d_view( spaceDomainBufferGPU, param_->total_num_z_sticks() * param_->num_xy_planes(comm_.rank()))), freqDomainBufferGPU_(create_new_type_1d_view( freqDomainBufferGPU, param_->num_z_sticks(comm_.rank()) * param_->total_num_xy_planes())), spaceDomainStream_(std::move(spaceDomainStream)), freqDomainStream_(std::move(freqDomainStream)) { assert(param_->dim_y() == spaceDomainDataGPU.dim_mid()); assert(param_->dim_x_freq() == spaceDomainDataGPU.dim_inner()); assert(param_->num_xy_planes(comm_.rank()) == spaceDomainDataGPU.dim_outer()); assert(param_->dim_z() == freqDomainDataGPU.dim_inner()); assert(param_->num_z_sticks(comm_.rank()) == freqDomainDataGPU.dim_outer()); assert(spaceDomainBufferGPU.size() >= param_->total_num_z_sticks() * param_->num_xy_planes(comm_.rank())); assert(spaceDomainBufferHost.size() >= param_->total_num_z_sticks() * param_->num_xy_planes(comm_.rank())); assert(freqDomainBufferGPU.size() >= param_->total_num_xy_planes() * param_->num_z_sticks(comm_.rank())); assert(freqDomainBufferHost.size() >= param_->total_num_xy_planes() * param_->num_z_sticks(comm_.rank())); // assert(disjoint(spaceDomainDataGPU, freqDomainDataGPU)); assert(disjoint(spaceDomainDataGPU, spaceDomainBufferGPU)); assert(disjoint(freqDomainDataGPU, freqDomainBufferGPU)); assert(disjoint(spaceDomainBufferHost, freqDomainBufferHost)); #ifdef SPFFT_GPU_DIRECT assert(disjoint(spaceDomainBufferGPU, freqDomainBufferGPU)); #endif // create underlying type mpiTypeHandle_ = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); // prepare mpi parameters spaceDomainCount_.resize(comm_.size()); freqDomainCount_.resize(comm_.size()); const SizeType numLocalZSticks = param_->num_z_sticks(comm_.rank()); const SizeType numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { freqDomainCount_[r] = numLocalZSticks * param_->num_xy_planes(r); spaceDomainCount_[r] = param_->num_z_sticks(r) * numLocalXYPlanes; } spaceDomainDispls_.resize(comm_.size()); freqDomainDispls_.resize(comm_.size()); int currentFreqDomainDispls = 0; int currentSpaceDomainDispls = 0; for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { assert(currentSpaceDomainDispls + spaceDomainCount_[r] <= static_cast(spaceDomainBufferHost.size())); assert(currentFreqDomainDispls + freqDomainCount_[r] <= static_cast(freqDomainBufferHost.size())); spaceDomainDispls_[r] = currentSpaceDomainDispls; freqDomainDispls_[r] = currentFreqDomainDispls; currentSpaceDomainDispls += spaceDomainCount_[r]; currentFreqDomainDispls += freqDomainCount_[r]; } // copy relevant parameters to gpu std::vector numZSticksHost(comm_.size()); std::vector numXYPlanesHost(comm_.size()); std::vector xyPlaneOffsetsHost(comm_.size()); std::vector indicesHost(comm_.size() * param_->max_num_z_sticks()); for (SizeType r = 0; r < comm_.size(); ++r) { numZSticksHost[r] = static_cast(param_->num_z_sticks(r)); numXYPlanesHost[r] = static_cast(param_->num_xy_planes(r)); xyPlaneOffsetsHost[r] = static_cast(param_->xy_plane_offset(r)); const auto zStickXYIndices = param_->z_stick_xy_indices(r); for (SizeType i = 0; i < zStickXYIndices.size(); ++i) { // transpose stick index const int xyIndex = zStickXYIndices(i); const int x = xyIndex / param_->dim_y(); const int y = xyIndex - x * param_->dim_y(); indicesHost[r * param_->max_num_z_sticks() + i] = y * param_->dim_x_freq() + x; } } numZSticksGPU_ = GPUArray(numZSticksHost.size()); numXYPlanesGPU_ = GPUArray(numXYPlanesHost.size()); xyPlaneOffsetsGPU_ = GPUArray(xyPlaneOffsetsHost.size()); indicesGPU_ = GPUArray(indicesHost.size()); copy_to_gpu(numZSticksHost, numZSticksGPU_); copy_to_gpu(numXYPlanesHost, numXYPlanesGPU_); copy_to_gpu(xyPlaneOffsetsHost, xyPlaneOffsetsGPU_); copy_to_gpu(indicesHost, indicesGPU_); } template auto TransposeMPICompactBufferedGPU::pack_backward() -> void { if (freqDomainDataGPU_.size() > 0 && freqDomainBufferGPU_.size() > 0) { compact_buffered_pack_backward(freqDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numXYPlanesGPU_, 0, numXYPlanesGPU_.size()), create_1d_view(xyPlaneOffsetsGPU_, 0, xyPlaneOffsetsGPU_.size()), freqDomainDataGPU_, freqDomainBufferGPU_); #ifndef SPFFT_GPU_DIRECT copy_from_gpu_async(freqDomainStream_, freqDomainBufferGPU_, freqDomainBufferHost_); #endif } } template auto TransposeMPICompactBufferedGPU::unpack_backward() -> void { if (spaceDomainDataGPU_.size() > 0) { gpu::check_status(gpu::memset_async( static_cast(spaceDomainDataGPU_.data()), 0, spaceDomainDataGPU_.size() * sizeof(typename decltype(spaceDomainDataGPU_)::ValueType), spaceDomainStream_.get())); if (spaceDomainBufferGPU_.size() > 0) { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(spaceDomainStream_, spaceDomainBufferHost_, spaceDomainBufferGPU_); #endif compact_buffered_unpack_backward(spaceDomainStream_.get(), param_->max_num_z_sticks(), create_1d_view(numZSticksGPU_, 0, numZSticksGPU_.size()), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), spaceDomainBufferGPU_, spaceDomainDataGPU_); } } } template auto TransposeMPICompactBufferedGPU::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(freqDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = freqDomainBufferGPU_.data(); auto recvBufferPtr = spaceDomainBufferGPU_.data(); #else auto sendBufferPtr = freqDomainBufferHost_.data(); auto recvBufferPtr = spaceDomainBufferHost_.data(); #endif if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallv( sendBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), recvBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallv(sendBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), recvBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPICompactBufferedGPU::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPICompactBufferedGPU::pack_forward() -> void { if (spaceDomainDataGPU_.size() > 0 && spaceDomainBufferGPU_.size() > 0) { compact_buffered_pack_forward(spaceDomainStream_.get(), param_->max_num_z_sticks(), create_1d_view(numZSticksGPU_, 0, numZSticksGPU_.size()), create_1d_view(indicesGPU_, 0, indicesGPU_.size()), spaceDomainDataGPU_, spaceDomainBufferGPU_); #ifndef SPFFT_GPU_DIRECT copy_from_gpu_async(spaceDomainStream_, spaceDomainBufferGPU_, spaceDomainBufferHost_); #endif } } template auto TransposeMPICompactBufferedGPU::unpack_forward() -> void { if (freqDomainDataGPU_.size() > 0 && freqDomainBufferGPU_.size() > 0) { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(freqDomainStream_, freqDomainBufferHost_, freqDomainBufferGPU_); #endif compact_buffered_unpack_forward( freqDomainStream_.get(), param_->max_num_xy_planes(), create_1d_view(numXYPlanesGPU_, 0, numXYPlanesGPU_.size()), create_1d_view(xyPlaneOffsetsGPU_, 0, xyPlaneOffsetsGPU_.size()), freqDomainBufferGPU_, freqDomainDataGPU_); } } template auto TransposeMPICompactBufferedGPU::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(spaceDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = spaceDomainBufferGPU_.data(); auto recvBufferPtr = freqDomainBufferGPU_.data(); #else auto sendBufferPtr = spaceDomainBufferHost_.data(); auto recvBufferPtr = freqDomainBufferHost_.data(); #endif if (nonBlockingExchange) { // start non-blocking exchange mpi_check_status(MPI_Ialltoallv( sendBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), recvBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { // blocking exchange mpi_check_status(MPI_Alltoallv(sendBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), recvBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPICompactBufferedGPU::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPICompactBufferedGPU; #endif template class TransposeMPICompactBufferedGPU; template class TransposeMPICompactBufferedGPU; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_compact_buffered_gpu.hpp000066400000000000000000000122501357247005600251600ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_COMPACT_BUFFERED_GPU_HPP #define SPFFT_TRANSPOSE_MPI_COMPACT_BUFFERED_GPU_HPP #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #if defined(SPFFT_MPI) && (defined(SPFFT_CUDA) || defined(SPFFT_ROCM)) #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPICompactBufferedGPU : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; using ComplexExchangeType = std::complex; using ComplexGPUType = typename gpu::fft::ComplexType::type; using ComplexExchangeGPUType = typename gpu::fft::ComplexType::type; public: // spaceDomainDataGPU and freqDomainDataGPU must NOT overlap // spaceDomainDataGPU and spaceDomainBufferGPU must NOT overlap // freqDomainDataGPU and freqDomainBufferGPU must NOT overlap // spaceDomainBufferGPU and freqDomainBufferGPU must NOT overlap // spaceDomainBufferHost and freqDomainBufferHost must NOT overlap // // spaceDomainBufferGPU and freqDomainDataGPU MAY overlap // freqDomainBufferGPU and spaceDomainDataGPU MAY overlap TransposeMPICompactBufferedGPU(const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView1D spaceDomainBufferHost, GPUArrayView3D spaceDomainDataGPU, GPUArrayView1D spaceDomainBufferGPU, GPUStreamHandle spaceDomainStream, HostArrayView1D freqDomainBufferHost, GPUArrayView2D freqDomainDataGPU, GPUArrayView1D freqDomainBufferGPU, GPUStreamHandle freqDomainStream); auto pack_backward() -> void override; auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto unpack_backward() -> void override; auto pack_forward() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; auto unpack_forward() -> void override; private: std::shared_ptr param_; MPIDatatypeHandle mpiTypeHandle_; MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; std::vector spaceDomainDispls_; std::vector freqDomainDispls_; std::vector spaceDomainCount_; std::vector freqDomainCount_; HostArrayView1D spaceDomainBufferHost_; HostArrayView1D freqDomainBufferHost_; GPUArrayView3D spaceDomainDataGPU_; GPUArrayView2D freqDomainDataGPU_; GPUArrayView1D spaceDomainBufferGPU_; GPUArrayView1D freqDomainBufferGPU_; GPUStreamHandle spaceDomainStream_; GPUStreamHandle freqDomainStream_; GPUArray numZSticksGPU_; GPUArray numXYPlanesGPU_; GPUArray xyPlaneOffsetsGPU_; GPUArray indicesGPU_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/transpose/transpose_mpi_compact_buffered_host.cpp000066400000000000000000000340621357247005600253420ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/transpose_mpi_compact_buffered_host.hpp" namespace spfft { template TransposeMPICompactBufferedHost::TransposeMPICompactBufferedHost( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData, HostArrayView1D spaceDomainBuffer, HostArrayView1D freqDomainBuffer) : param_(param), comm_(std::move(comm)), spaceDomainData_(spaceDomainData), freqDomainData_(freqDomainData), spaceDomainBuffer_(create_new_type_1d_view(spaceDomainBuffer, spaceDomainBuffer.size())), freqDomainBuffer_( create_new_type_1d_view(freqDomainBuffer, freqDomainBuffer.size())) { assert(param_->dim_x_freq() == spaceDomainData.dim_mid()); assert(param_->dim_y() == spaceDomainData.dim_inner()); assert(param_->num_xy_planes(comm_.rank()) == spaceDomainData.dim_outer()); assert(param_->dim_z() == freqDomainData.dim_inner()); assert(param_->num_z_sticks(comm_.rank()) == freqDomainData.dim_outer()); assert(spaceDomainBuffer.size() >= param_->total_num_z_sticks() * param_->num_xy_planes(comm_.rank())); assert(freqDomainBuffer.size() >= param_->total_num_xy_planes() * param_->num_z_sticks(comm_.rank())); assert(disjoint(spaceDomainData, freqDomainData)); assert(disjoint(spaceDomainData, spaceDomainBuffer)); assert(disjoint(freqDomainData, freqDomainBuffer)); assert(disjoint(spaceDomainBuffer, freqDomainBuffer)); // create underlying type mpiTypeHandle_ = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); spaceDomainCount_.resize(comm_.size()); freqDomainCount_.resize(comm_.size()); const SizeType numLocalZSticks = param_->num_z_sticks(comm_.rank()); const SizeType numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < comm_.size(); ++r) { freqDomainCount_[r] = numLocalZSticks * param_->num_xy_planes(r); spaceDomainCount_[r] = param_->num_z_sticks(r) * numLocalXYPlanes; } spaceDomainDispls_.resize(comm_.size()); freqDomainDispls_.resize(comm_.size()); int currentFreqDomainDispls = 0; int currentSpaceDomainDispls = 0; for (SizeType r = 0; r < comm_.size(); ++r) { assert(currentSpaceDomainDispls + spaceDomainCount_[r] <= static_cast(spaceDomainBuffer.size())); assert(currentFreqDomainDispls + freqDomainCount_[r] <= static_cast(freqDomainBuffer.size())); spaceDomainDispls_[r] = currentSpaceDomainDispls; freqDomainDispls_[r] = currentFreqDomainDispls; currentSpaceDomainDispls += spaceDomainCount_[r]; currentFreqDomainDispls += freqDomainCount_[r]; } } template auto TransposeMPICompactBufferedHost::pack_backward() -> void { // transpose locally from (numLocalZSticks, dimZ) to (dimZ, numLocalZSticks) for (SizeType r = 0; r < static_cast(comm_.size()); ++r) { const auto xyPlaneOffset = param_->xy_plane_offset(r); auto freqDomainBuffer2d = create_2d_view(freqDomainBuffer_, freqDomainDispls_[r], freqDomainData_.dim_outer(), param_->num_xy_planes(r)); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < freqDomainData_.dim_outer(); ++zStickIndex) { for (SizeType zIndex = 0; zIndex < param_->num_xy_planes(r); ++zIndex) { freqDomainBuffer2d(zStickIndex, zIndex) = freqDomainData_(zStickIndex, zIndex + xyPlaneOffset); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPICompactBufferedHost::unpack_backward() -> void { // zero target data location (not all values are overwritten upon unpacking) SPFFT_OMP_PRAGMA("omp for schedule(static)") // implicit barrier for (SizeType z = 0; z < spaceDomainData_.dim_outer(); ++z) { std::memset(static_cast(&spaceDomainData_(z, 0, 0)), 0, sizeof(typename decltype(spaceDomainData_)::ValueType) * spaceDomainData_.dim_inner() * spaceDomainData_.dim_mid()); } auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // unpack from (numZSticksTotal, numLocalXYPlanes) to (numLocalXYPlanes, dimX, dimY) const auto numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { const auto zStickXYIndices = param_->z_stick_xy_indices(r); // take care with unsigned type const SizeType unrolledLoopEnd = zStickXYIndices.size() < 4 ? 0 : zStickXYIndices.size() - 3; auto recvBuffer = create_2d_view(spaceDomainBuffer_, spaceDomainDispls_[r], zStickXYIndices.size(), numLocalXYPlanes); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { const SizeType xyIndex1 = zStickXYIndices(zStickIndex); const SizeType xyIndex2 = zStickXYIndices(zStickIndex + 1); const SizeType xyIndex3 = zStickXYIndices(zStickIndex + 2); const SizeType xyIndex4 = zStickXYIndices(zStickIndex + 3); // manual loop unrolling for better performance for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex1) = recvBuffer(zStickIndex, zIndex); spaceDomainDataFlat(zIndex, xyIndex2) = recvBuffer(zStickIndex + 1, zIndex); spaceDomainDataFlat(zIndex, xyIndex3) = recvBuffer(zStickIndex + 2, zIndex); spaceDomainDataFlat(zIndex, xyIndex4) = recvBuffer(zStickIndex + 3, zIndex); } } SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < zStickXYIndices.size(); zStickIndex += 1) { const SizeType xyIndex = zStickXYIndices(zStickIndex); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { spaceDomainDataFlat(zIndex, xyIndex) = recvBuffer(zStickIndex, zIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPICompactBufferedHost::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPICompactBufferedHost::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter // exchange data if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallv(freqDomainBuffer_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), spaceDomainBuffer_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallv(freqDomainBuffer_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), spaceDomainBuffer_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get())); } } template auto TransposeMPICompactBufferedHost::pack_forward() -> void { auto spaceDomainDataFlat = create_2d_view(spaceDomainData_, 0, spaceDomainData_.dim_outer(), spaceDomainData_.dim_mid() * spaceDomainData_.dim_inner()); // pack from (numLocalXYPlanes, dimX, dimY) to (numZSticksTotal, numLocalXYPlanes) const auto numLocalXYPlanes = param_->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < (SizeType)comm_.size(); ++r) { const auto zStickXYIndices = param_->z_stick_xy_indices(r); // take care with unsigned type const SizeType unrolledLoopEnd = zStickXYIndices.size() < 4 ? 0 : zStickXYIndices.size() - 3; auto recvBuffer = create_2d_view(spaceDomainBuffer_, spaceDomainDispls_[r], zStickXYIndices.size(), numLocalXYPlanes); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < unrolledLoopEnd; zStickIndex += 4) { // manual loop unrolling for better performance const SizeType xyIndex1 = zStickXYIndices(zStickIndex); const SizeType xyIndex2 = zStickXYIndices(zStickIndex + 1); const SizeType xyIndex3 = zStickXYIndices(zStickIndex + 2); const SizeType xyIndex4 = zStickXYIndices(zStickIndex + 3); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { recvBuffer(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex1); recvBuffer(zStickIndex + 1, zIndex) = spaceDomainDataFlat(zIndex, xyIndex2); recvBuffer(zStickIndex + 2, zIndex) = spaceDomainDataFlat(zIndex, xyIndex3); recvBuffer(zStickIndex + 3, zIndex) = spaceDomainDataFlat(zIndex, xyIndex4); } } SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = unrolledLoopEnd; zStickIndex < zStickXYIndices.size(); zStickIndex += 1) { const SizeType xyIndex = zStickXYIndices(zStickIndex); for (SizeType zIndex = 0; zIndex < numLocalXYPlanes; ++zIndex) { recvBuffer(zStickIndex, zIndex) = spaceDomainDataFlat(zIndex, xyIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPICompactBufferedHost::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPICompactBufferedHost::unpack_forward() -> void { // transpose locally from (dimZ, numLocalZSticks) to (numLocalZSticks, dimZ) for (SizeType r = 0; r < static_cast(comm_.size()); ++r) { const auto xyPlaneOffset = param_->xy_plane_offset(r); auto freqDomainBuffer2d = create_2d_view(freqDomainBuffer_, freqDomainDispls_[r], freqDomainData_.dim_outer(), param_->num_xy_planes(r)); SPFFT_OMP_PRAGMA("omp for schedule(static) nowait") for (SizeType zStickIndex = 0; zStickIndex < freqDomainData_.dim_outer(); ++zStickIndex) { for (SizeType zIndex = 0; zIndex < param_->num_xy_planes(r); ++zIndex) { freqDomainData_(zStickIndex, zIndex + xyPlaneOffset) = freqDomainBuffer2d(zStickIndex, zIndex); } } } SPFFT_OMP_PRAGMA("omp barrier") } template auto TransposeMPICompactBufferedHost::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallv(spaceDomainBuffer_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), freqDomainBuffer_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallv(spaceDomainBuffer_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), mpiTypeHandle_.get(), freqDomainBuffer_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), mpiTypeHandle_.get(), comm_.get())); } } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPICompactBufferedHost; #endif template class TransposeMPICompactBufferedHost; template class TransposeMPICompactBufferedHost; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_compact_buffered_host.hpp000066400000000000000000000100571357247005600253450ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_COMPACT_BUFFERED_HOST_HPP #define SPFFT_TRANSPOSE_MPI_COMPACT_BUFFERED_HOST_HPP #include #include #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPICompactBufferedHost : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; using ComplexExchangeType = std::complex; public: // spaceDomainData and freqDomainData must NOT overlap // spaceDomainData and spaceDomainBuffer must NOT overlap // freqDomainData and freqDomainBuffer must NOT overlap // spaceDomainBuffer and freqDomainBuffer must NOT overlap // // spaceDomainBuffer and freqDomainData MAY overlap // freqDomainBuffer and spaceDomainData MAY overlap TransposeMPICompactBufferedHost(const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData, HostArrayView1D spaceDomainBuffer, HostArrayView1D freqDomainBuffer); auto pack_backward() -> void override; auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto unpack_backward() -> void override; auto pack_forward() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; auto unpack_forward() -> void override; private: std::shared_ptr param_; MPIDatatypeHandle mpiTypeHandle_; MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; HostArrayView3D spaceDomainData_; HostArrayView2D freqDomainData_; HostArrayView1D spaceDomainBuffer_; HostArrayView1D freqDomainBuffer_; std::vector spaceDomainDispls_; std::vector freqDomainDispls_; std::vector spaceDomainCount_; std::vector freqDomainCount_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/transpose/transpose_mpi_unbuffered_gpu.cpp000066400000000000000000000256311357247005600240170ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include "gpu_util/gpu_transfer.hpp" #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/transpose_mpi_unbuffered_gpu.hpp" namespace spfft { template TransposeMPIUnbufferedGPU::TransposeMPIUnbufferedGPU( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, GPUArrayView3D::type> spaceDomainDataGPU, GPUStreamHandle spaceDomainStream, HostArrayView2D freqDomainData, GPUArrayView2D::type> freqDomainDataGPU, GPUStreamHandle freqDomainStream) : comm_(std::move(comm)), spaceDomainBufferHost_(spaceDomainData), freqDomainBufferHost_(freqDomainData), spaceDomainBufferGPU_(spaceDomainDataGPU), freqDomainBufferGPU_(freqDomainDataGPU), numLocalXYPlanes_(spaceDomainData.dim_outer()), spaceDomainStream_(std::move(spaceDomainStream)), freqDomainStream_(std::move(freqDomainStream)) { assert(disjoint(spaceDomainData, freqDomainData)); assert(param->dim_x_freq() == spaceDomainData.dim_inner()); assert(param->dim_y() == spaceDomainData.dim_mid()); assert(param->num_xy_planes(comm_.rank()) == spaceDomainData.dim_outer()); assert(param->dim_z() == freqDomainData.dim_inner()); assert(param->num_z_sticks(comm_.rank()) == freqDomainData.dim_outer()); // create underlying type MPIDatatypeHandle complexType = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); // create types in frequency space for each rank: // each type represents a fixed length part of every z stick the rank holds freqDomainTypeHandles_.reserve(comm_.size()); freqDomainCount_.reserve(comm_.size()); freqDomainTypes_.reserve(comm_.size()); freqDomainDispls_.assign(comm_.size(), 0); const SizeType numLocalZSticks = param->num_z_sticks(comm_.rank()); const SizeType numLocalXYPlanes = param->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < comm_.size(); ++r) { if (param->num_xy_planes(r) > 0 && numLocalZSticks > 0) { const int ndims = 2; const int arrayOfSizes[] = {(int)numLocalZSticks, (int)freqDomainBufferHost_.dim_inner()}; const int arrayOfSubsizes[] = {(int)numLocalZSticks, (int)param->num_xy_planes(r)}; const int arrayOfStarts[] = {(int)0, (int)param->xy_plane_offset(r)}; const int order = MPI_ORDER_C; freqDomainCount_.emplace_back(1); freqDomainTypeHandles_.emplace_back(MPIDatatypeHandle::create_subarray( ndims, arrayOfSizes, arrayOfSubsizes, arrayOfStarts, order, complexType.get())); freqDomainTypes_.emplace_back(freqDomainTypeHandles_.back().get()); } else { freqDomainCount_.emplace_back(0); freqDomainTypeHandles_.emplace_back(complexType); freqDomainTypes_.emplace_back(freqDomainTypeHandles_.back().get()); } } // create types in space domain for each rank: // each type represents a batch of partial z sticks with inner stride dimX*dimY and placed // according to the assosiated x/y indices std::vector indexedBlocklengths; std::vector indexedDispls; spaceDomainTypes_.reserve(comm_.size()); spaceDomainCount_.reserve(comm_.size()); spaceDomainDispls_.assign(comm_.size(), 0); for (SizeType r = 0; r < comm_.size(); ++r) { if (param->num_z_sticks(r) > 0 && numLocalXYPlanes > 0) { // data type for single z stick part MPIDatatypeHandle stridedZStickType = MPIDatatypeHandle::create_vector( numLocalXYPlanes, 1, spaceDomainBufferHost_.dim_inner() * spaceDomainBufferHost_.dim_mid(), complexType.get()); const auto zStickXYIndices = param->z_stick_xy_indices(r); indexedBlocklengths.resize(zStickXYIndices.size(), 1); indexedDispls.resize(zStickXYIndices.size()); // displacements of all z stick parts to be send to current rank for (SizeType idxZStick = 0; idxZStick < zStickXYIndices.size(); ++idxZStick) { // transpose stick index const int xyIndex = zStickXYIndices(idxZStick); const int x = xyIndex / param->dim_y(); const int y = xyIndex - x * param->dim_y(); indexedDispls[idxZStick] = 2 * sizeof(T) * (y * param->dim_x_freq() + x); } spaceDomainCount_.emplace_back(1); spaceDomainTypeHandles_.emplace_back( MPIDatatypeHandle::create_hindexed(zStickXYIndices.size(), indexedBlocklengths.data(), indexedDispls.data(), stridedZStickType.get())); spaceDomainTypes_.emplace_back(spaceDomainTypeHandles_.back().get()); } else { spaceDomainCount_.emplace_back(0); spaceDomainTypeHandles_.emplace_back(complexType); spaceDomainTypes_.emplace_back(complexType.get()); } } } template auto TransposeMPIUnbufferedGPU::pack_backward() -> void { #ifdef SPFFT_GPU_DIRECT gpu::check_status(gpu::memset_async( static_cast(spaceDomainBufferGPU_.data()), 0, spaceDomainBufferGPU_.size() * sizeof(typename decltype(spaceDomainBufferGPU_)::ValueType), spaceDomainStream_.get())); #else copy_from_gpu_async(freqDomainStream_, freqDomainBufferGPU_, freqDomainBufferHost_); // zero target data location (not all values are overwritten upon unpacking) std::memset( static_cast(spaceDomainBufferHost_.data()), 0, sizeof(typename decltype(spaceDomainBufferHost_)::ValueType) * spaceDomainBufferHost_.size()); #endif } template auto TransposeMPIUnbufferedGPU::unpack_backward() -> void { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(spaceDomainStream_, spaceDomainBufferHost_, spaceDomainBufferGPU_); #endif } template auto TransposeMPIUnbufferedGPU::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(freqDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = freqDomainBufferGPU_.data(); auto recvBufferPtr = spaceDomainBufferGPU_.data(); #else auto sendBufferPtr = freqDomainBufferHost_.data(); auto recvBufferPtr = spaceDomainBufferHost_.data(); #endif if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallw( sendBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), recvBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallw(sendBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), recvBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), comm_.get())); } } template auto TransposeMPIUnbufferedGPU::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPIUnbufferedGPU::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter gpu::check_status(gpu::stream_synchronize(spaceDomainStream_.get())); #ifdef SPFFT_GPU_DIRECT auto sendBufferPtr = spaceDomainBufferGPU_.data(); auto recvBufferPtr = freqDomainBufferGPU_.data(); #else auto sendBufferPtr = spaceDomainBufferHost_.data(); auto recvBufferPtr = freqDomainBufferHost_.data(); #endif if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallw( sendBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), recvBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallw(sendBufferPtr, spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), recvBufferPtr, freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), comm_.get())); } } template auto TransposeMPIUnbufferedGPU::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPIUnbufferedGPU::pack_forward() -> void { #ifndef SPFFT_GPU_DIRECT copy_from_gpu_async(spaceDomainStream_, spaceDomainBufferGPU_, spaceDomainBufferHost_); #endif } template auto TransposeMPIUnbufferedGPU::unpack_forward() -> void { #ifndef SPFFT_GPU_DIRECT copy_to_gpu_async(freqDomainStream_, freqDomainBufferHost_, freqDomainBufferGPU_); #endif } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPIUnbufferedGPU; #endif template class TransposeMPIUnbufferedGPU; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_unbuffered_gpu.hpp000066400000000000000000000101621357247005600240150ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_UNBUFFERED_GPU_HPP #define SPFFT_TRANSPOSE_MPI_UNBUFFERED_GPU_HPP #include #include #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_stream_handle.hpp" #include "memory/gpu_array.hpp" #include "memory/gpu_array_view.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPIUnbufferedGPU : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; public: TransposeMPIUnbufferedGPU( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, GPUArrayView3D::type> spaceDomainDataGPU, GPUStreamHandle spaceDomainStream, HostArrayView2D freqDomainData, GPUArrayView2D::type> freqDomainDataGPU, GPUStreamHandle freqDomainStream); auto pack_backward() -> void override; auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto unpack_backward() -> void override; auto pack_forward() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; auto unpack_forward() -> void override; private: MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; HostArrayView3D spaceDomainBufferHost_; HostArrayView2D freqDomainBufferHost_; GPUArrayView3D::type> spaceDomainBufferGPU_; GPUArrayView2D::type> freqDomainBufferGPU_; SizeType numLocalXYPlanes_; GPUStreamHandle spaceDomainStream_; GPUStreamHandle freqDomainStream_; std::vector freqDomainTypeHandles_; std::vector freqDomainTypes_; std::vector freqDomainDispls_; std::vector freqDomainCount_; std::vector spaceDomainTypeHandles_; std::vector spaceDomainTypes_; std::vector spaceDomainDispls_; std::vector spaceDomainCount_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/transpose/transpose_mpi_unbuffered_host.cpp000066400000000000000000000215151357247005600241760ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include "memory/array_view_utility.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/exceptions.hpp" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/omp_definitions.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_check_status.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_match_elementary_type.hpp" #include "transpose/transpose_mpi_unbuffered_host.hpp" namespace spfft { template TransposeMPIUnbufferedHost::TransposeMPIUnbufferedHost( const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData) : comm_(std::move(comm)), spaceDomainData_(spaceDomainData), freqDomainData_(freqDomainData), numLocalXYPlanes_(spaceDomainData.dim_outer()) { assert(disjoint(spaceDomainData, freqDomainData)); assert(param->dim_x_freq() == spaceDomainData.dim_mid()); assert(param->dim_y() == spaceDomainData.dim_inner()); assert(param->num_xy_planes(comm_.rank()) == spaceDomainData.dim_outer()); assert(param->dim_z() == freqDomainData.dim_inner()); assert(param->num_z_sticks(comm_.rank()) == freqDomainData.dim_outer()); // create underlying type MPIDatatypeHandle complexType = MPIDatatypeHandle::create_contiguous(2, MPIMatchElementaryType::get()); // create types in frequency space for each rank: // each type represents a fixed length part of every z stick the rank holds freqDomainTypeHandles_.reserve(comm_.size()); freqDomainCount_.reserve(comm_.size()); freqDomainTypes_.reserve(comm_.size()); freqDomainDispls_.assign(comm_.size(), 0); const SizeType numLocalZSticks = param->num_z_sticks(comm_.rank()); const SizeType numLocalXYPlanes = param->num_xy_planes(comm_.rank()); for (SizeType r = 0; r < comm_.size(); ++r) { if (param->num_xy_planes(r) > 0 && numLocalZSticks > 0) { const int ndims = 2; const int arrayOfSizes[] = {(int)numLocalZSticks, (int)freqDomainData_.dim_inner()}; const int arrayOfSubsizes[] = {(int)numLocalZSticks, (int)param->num_xy_planes(r)}; const int arrayOfStarts[] = {(int)0, (int)param->xy_plane_offset(r)}; const int order = MPI_ORDER_C; freqDomainCount_.emplace_back(1); freqDomainTypeHandles_.emplace_back(MPIDatatypeHandle::create_subarray( ndims, arrayOfSizes, arrayOfSubsizes, arrayOfStarts, order, complexType.get())); freqDomainTypes_.emplace_back(freqDomainTypeHandles_.back().get()); } else { freqDomainCount_.emplace_back(0); freqDomainTypeHandles_.emplace_back(complexType); freqDomainTypes_.emplace_back(freqDomainTypeHandles_.back().get()); } } // create types in space domain for each rank: // each type represents a batch of partial z sticks with inner stride dimX*dimY and placed // according to the assosiated x/y indices std::vector indexedBlocklengths; std::vector indexedDispls; spaceDomainTypes_.reserve(comm_.size()); spaceDomainCount_.reserve(comm_.size()); spaceDomainDispls_.assign(comm_.size(), 0); for (SizeType r = 0; r < comm_.size(); ++r) { if (param->num_z_sticks(r) > 0 && numLocalXYPlanes > 0) { // data type for single z stick part MPIDatatypeHandle stridedZStickType = MPIDatatypeHandle::create_vector( numLocalXYPlanes, 1, spaceDomainData_.dim_inner() * spaceDomainData_.dim_mid(), complexType.get()); const auto zStickXYIndices = param->z_stick_xy_indices(r); indexedBlocklengths.resize(zStickXYIndices.size(), 1); indexedDispls.resize(zStickXYIndices.size()); // displacements of all z stick parts to be send to current rank for (SizeType idxZStick = 0; idxZStick < zStickXYIndices.size(); ++idxZStick) { const auto& xyIndex = zStickXYIndices(idxZStick); indexedDispls[idxZStick] = 2 * sizeof(T) * xyIndex; } spaceDomainCount_.emplace_back(1); spaceDomainTypeHandles_.emplace_back( MPIDatatypeHandle::create_hindexed(zStickXYIndices.size(), indexedBlocklengths.data(), indexedDispls.data(), stridedZStickType.get())); spaceDomainTypes_.emplace_back(spaceDomainTypeHandles_.back().get()); } else { spaceDomainCount_.emplace_back(0); spaceDomainTypeHandles_.emplace_back(complexType); spaceDomainTypes_.emplace_back(complexType.get()); } } } template auto TransposeMPIUnbufferedHost::exchange_backward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter // zero target data location (not all values are overwritten upon unpacking) std::memset(static_cast(spaceDomainData_.data()), 0, sizeof(typename decltype(spaceDomainData_)::ValueType) * spaceDomainData_.size()); if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallw(freqDomainData_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), spaceDomainData_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status( MPI_Alltoallw(freqDomainData_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), spaceDomainData_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), comm_.get())); } } template auto TransposeMPIUnbufferedHost::exchange_backward_finalize() -> void { mpiRequest_.wait_if_active(); } template auto TransposeMPIUnbufferedHost::exchange_forward_start(const bool nonBlockingExchange) -> void { assert(omp_get_thread_num() == 0); // only must thread must be allowed to enter if (nonBlockingExchange) { mpi_check_status(MPI_Ialltoallw(spaceDomainData_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), freqDomainData_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), comm_.get(), mpiRequest_.get_and_activate())); } else { mpi_check_status(MPI_Alltoallw(spaceDomainData_.data(), spaceDomainCount_.data(), spaceDomainDispls_.data(), spaceDomainTypes_.data(), freqDomainData_.data(), freqDomainCount_.data(), freqDomainDispls_.data(), freqDomainTypes_.data(), comm_.get())); } } template auto TransposeMPIUnbufferedHost::exchange_forward_finalize() -> void { mpiRequest_.wait_if_active(); } // Instantiate class for float and double #ifdef SPFFT_SINGLE_PRECISION template class TransposeMPIUnbufferedHost; #endif template class TransposeMPIUnbufferedHost; } // namespace spfft #endif // SPFFT_MPI SpFFT-0.9.10/src/transpose/transpose_mpi_unbuffered_host.hpp000066400000000000000000000065351357247005600242100ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TRANSPOSE_MPI_UNBUFFERED_HOST_HPP #define SPFFT_TRANSPOSE_MPI_UNBUFFERED_HOST_HPP #include #include #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/config.h" #include "transpose.hpp" #include "util/common_types.hpp" #include "util/type_check.hpp" #ifdef SPFFT_MPI #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_datatype_handle.hpp" #include "mpi_util/mpi_request_handle.hpp" namespace spfft { template class TransposeMPIUnbufferedHost : public Transpose { static_assert(IsFloatOrDouble::value, "Type T must be float or double"); using ValueType = T; using ComplexType = std::complex; public: TransposeMPIUnbufferedHost(const std::shared_ptr& param, MPICommunicatorHandle comm, HostArrayView3D spaceDomainData, HostArrayView2D freqDomainData); auto exchange_backward_start(const bool nonBlockingExchange) -> void override; auto exchange_backward_finalize() -> void override; auto exchange_forward_start(const bool nonBlockingExchange) -> void override; auto exchange_forward_finalize() -> void override; private: MPICommunicatorHandle comm_; MPIRequestHandle mpiRequest_; HostArrayView3D spaceDomainData_; HostArrayView2D freqDomainData_; SizeType numLocalXYPlanes_; std::vector freqDomainTypeHandles_; std::vector freqDomainTypes_; std::vector freqDomainDispls_; std::vector freqDomainCount_; std::vector spaceDomainTypeHandles_; std::vector spaceDomainTypes_; std::vector spaceDomainDispls_; std::vector spaceDomainCount_; }; } // namespace spfft #endif // SPFFT_MPI #endif SpFFT-0.9.10/src/util/000077500000000000000000000000001357247005600143005ustar00rootroot00000000000000SpFFT-0.9.10/src/util/common_types.hpp000066400000000000000000000033511357247005600175270ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_COMMON_TYPES_HPP #define SPFFT_COMMON_TYPES_HPP #include "spfft/config.h" namespace spfft { typedef unsigned long long SizeType; typedef long long SignedType; } // namespace spfft #endif SpFFT-0.9.10/src/util/omp_definitions.hpp000066400000000000000000000042111357247005600201750ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_OMP_DEFINITIONS_HPP #define SPFFT_OMP_DEFINITIONS_HPP #include "spfft/config.h" #ifdef SPFFT_OMP #include #define SPFFT_OMP_PRAGMA(content) _Pragma(content) #else #define SPFFT_OMP_PRAGMA(content) namespace spfft { inline int omp_get_num_threads() { return 1; } inline int omp_get_thread_num() { return 0; } inline int omp_get_max_threads() { return 1; } inline int omp_in_parallel() { return 0; } inline int omp_get_nested() { return 0; } inline int omp_get_num_procs() { return 1; } inline int omp_get_level() { return 0; } inline void omp_set_nested(int) {} } // namespace spfft #endif #endif SpFFT-0.9.10/src/util/type_check.hpp000066400000000000000000000037001357247005600171270ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TYPE_CHECK #define SPFFT_TYPE_CHECK #include #include "spfft/config.h" namespace spfft { template struct IsFloatOrDouble : std::integral_constant::type>::value || std::is_same::type>::value> {}; } // namespace spfft #endif SpFFT-0.9.10/style_guide.md000066400000000000000000000021351357247005600153740ustar00rootroot00000000000000# Style Guide for SpFFT ## Formatting The formatting style is based on the google style guide with the following exceptions: - Column size is limited to 100 instead of 80 - Access modifiers such as public and private are offset by -2 Clang-Format is used to format all files. ## Naming The following rules are not strict and consistency when using external types is preferred. ### Files Use underscores for separation. File suffix: - C++: .cpp and .hpp - C: .c and .h - CUDA: .cu and .cuh Example `my_class.cpp` ### Types Use camelcase and start with a capital letter. Example `using MyType = int;` ### Variables Use camelcase and start with lower case Example: `int myValue = 0;` #### Class / Struct Members Use a trailing underscore for non-public member variables. Public members are mamed like normal variables. #### Functions Function names use underscores and are all lower case Example: `my_function(int);` #### namespaces Namepsace are all lower case and use underscores. Example: ` namespace my_space {}` #### Macros Macros are all capital and use underscores. Example: `#define MY_MACRO_VALUE 1` SpFFT-0.9.10/tests/000077500000000000000000000000001357247005600136765ustar00rootroot00000000000000SpFFT-0.9.10/tests/CMakeLists.txt000066400000000000000000000067651357247005600164540ustar00rootroot00000000000000 if(SPFFT_BUILD_TESTS) cmake_minimum_required(VERSION 3.11 FATAL_ERROR) # git fetch module requires at least 3.11 set(BUILD_GMOCK OFF CACHE BOOL "") set(INSTALL_GTEST OFF CACHE BOOL "") mark_as_advanced(BUILD_GMOCK INSTALL_GTEST) include(FetchContent) # add googletest FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-1.8.1 ) FetchContent_GetProperties(googletest) if(NOT googletest_POPULATED) message(STATUS "Downloading Google Test repository...") FetchContent_Populate(googletest) endif() add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) # add gtest_mpi FetchContent_Declare( gtest_mpi GIT_REPOSITORY https://github.com/AdhocMan/gtest_mpi.git GIT_TAG v1.0.0 ) FetchContent_GetProperties(gtest_mpi) if(NOT gtest_mpi_POPULATED) message(STATUS "Downloading Google Test MPI extension repository...") FetchContent_Populate(gtest_mpi) endif() add_subdirectory(${gtest_mpi_SOURCE_DIR} ${gtest_mpi_BINARY_DIR}) # add command line parser FetchContent_Declare( cli11 GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git GIT_TAG v1.7.1 ) FetchContent_GetProperties(cli11) if(NOT cli11_POPULATED) message(STATUS "Downloading CLI11 command line parser repository...") FetchContent_Populate(cli11) endif() list(APPEND SPFFT_EXTERNAL_INCLUDE_DIRS ${cli11_SOURCE_DIR}/include) # add json parser set(JSON_Install OFF CACHE BOOL "") FetchContent_Declare( json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG v3.6.1 ) FetchContent_GetProperties(json) if(NOT json_POPULATED) message(STATUS "Downloading json repository...") FetchContent_Populate(json) endif() set(JSON_BuildTests OFF CACHE INTERNAL "") add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR}) list(APPEND SPFFT_EXTERNAL_LIBS nlohmann_json::nlohmann_json) list(APPEND SPFFT_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/tests) # benchmark executable add_executable(benchmark programs/benchmark.cpp) target_link_libraries(benchmark PRIVATE ${SPFFT_LIBS} ${SPFFT_EXTERNAL_LIBS}) target_compile_options(benchmark PRIVATE ${SPFFT_DEFINITIONS} ${SPFFT_EXTERNAL_COMPILE_OPTIONS}) target_include_directories(benchmark PRIVATE ${SPFFT_INCLUDE_DIRS} ${SPFFT_EXTERNAL_INCLUDE_DIRS}) # if(SPFFT_CUDA) # set_property(TARGET main PROPERTY CUDA_SEPARABLE_COMPILATION ON) # endif() # test executables add_executable(run_local_tests run_local_tests.cpp local_tests/test_host_array.cpp local_tests/test_disjoint.cpp local_tests/test_local_transform.cpp ) target_link_libraries(run_local_tests PRIVATE gtest_main gtest_mpi) target_link_libraries(run_local_tests PRIVATE ${SPFFT_LIBS} ${SPFFT_EXTERNAL_LIBS}) target_compile_options(run_local_tests PRIVATE ${SPFFT_DEFINITIONS} ${SPFFT_EXTERNAL_COMPILE_OPTIONS}) target_include_directories(run_local_tests PRIVATE ${SPFFT_INCLUDE_DIRS} ${SPFFT_EXTERNAL_INCLUDE_DIRS}) if(SPFFT_MPI) add_executable(run_mpi_tests run_mpi_tests.cpp mpi_tests/test_transform.cpp mpi_tests/test_multi_transform.cpp mpi_tests/test_transpose.cpp mpi_tests/test_transpose_gpu.cpp ) target_link_libraries(run_mpi_tests PRIVATE gtest_main gtest_mpi) target_link_libraries(run_mpi_tests PRIVATE ${SPFFT_LIBS} ${SPFFT_EXTERNAL_LIBS}) target_compile_options(run_mpi_tests PRIVATE ${SPFFT_DEFINITIONS} ${SPFFT_EXTERNAL_COMPILE_OPTIONS}) target_include_directories(run_mpi_tests PRIVATE ${SPFFT_INCLUDE_DIRS} ${SPFFT_EXTERNAL_INCLUDE_DIRS}) endif() endif() SpFFT-0.9.10/tests/local_tests/000077500000000000000000000000001357247005600162125ustar00rootroot00000000000000SpFFT-0.9.10/tests/local_tests/test_disjoint.cpp000066400000000000000000000057241357247005600216100ustar00rootroot00000000000000#include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" using namespace spfft; class DisjointTest : public ::testing::Test { protected: void SetUp() override { array_ = HostArray(100); } HostArray array_; }; TEST_F(DisjointTest, dim1AndDim1) { { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_1d_view(array_, 0, 10); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_1d_view(array_, 5, 10); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_1d_view(array_, 10, 10); EXPECT_TRUE(disjoint(view1, view2)); } } TEST_F(DisjointTest, dim1AndDim2) { { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_2d_view(array_, 0, 2, 5); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_2d_view(array_, 5, 2, 5); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_2d_view(array_, 10, 5, 2); EXPECT_TRUE(disjoint(view1, view2)); } } TEST_F(DisjointTest, dim1AndDim3) { { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_3d_view(array_, 0, 2, 2, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_3d_view(array_, 5, 2, 2, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_1d_view(array_, 0, 10); auto view2 = create_3d_view(array_, 10, 5, 2, 2); EXPECT_TRUE(disjoint(view1, view2)); } } TEST_F(DisjointTest, dim2AndDim3) { { auto view1 = create_2d_view(array_, 0, 2, 3); auto view2 = create_3d_view(array_, 0, 2, 3, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_2d_view(array_, 0, 2, 3); auto view2 = create_3d_view(array_, 5, 2, 2, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_2d_view(array_, 0, 2, 3); auto view2 = create_3d_view(array_, 6, 5, 2, 2); EXPECT_TRUE(disjoint(view1, view2)); } } TEST_F(DisjointTest, dim3AndDim3) { { auto view1 = create_3d_view(array_, 0, 2, 3, 4); auto view2 = create_3d_view(array_, 0, 2, 3, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_3d_view(array_, 0, 2, 3, 4); auto view2 = create_3d_view(array_, 5, 2, 2, 2); EXPECT_FALSE(disjoint(view1, view2)); } { auto view1 = create_3d_view(array_, 0, 2, 3, 2); auto view2 = create_3d_view(array_, 12, 5, 2, 2); EXPECT_TRUE(disjoint(view1, view2)); } } TEST_F(DisjointTest, DifferentValueTypes) { auto view1 = create_3d_view(array_, 0, 2, 3, 4); auto view2 = HostArrayView3D(reinterpret_cast(array_.data()), 2, 3, 4, false); EXPECT_FALSE(disjoint(view1, view2)); } SpFFT-0.9.10/tests/local_tests/test_host_array.cpp000066400000000000000000000016631357247005600221360ustar00rootroot00000000000000#include #include "gtest/gtest.h" #include "memory/host_array.hpp" using namespace spfft; class HostArrayTest : public ::testing::Test { protected: void SetUp() override { array_ = HostArray(5); int count = 0; auto data_ptr = array_.data(); for (SizeType i = 0; i < 5; ++i) { data_ptr[i] = ++count; } } HostArray array_; }; TEST_F(HostArrayTest, Iterators) { ASSERT_EQ(*array_.begin(), 1); ASSERT_EQ(*(array_.end() - 1), 5); int count = 0; for (auto& val : array_) { EXPECT_EQ(val, ++count); } } TEST_F(HostArrayTest, OperatorAccess) { int count = 0; ASSERT_EQ(array_.size(), 5); for (SizeType i = 0; i < array_.size(); ++i) { ASSERT_EQ(array_[i], ++count); } count = 0; for (SizeType i = 0; i < array_.size(); ++i) { ASSERT_EQ(array_(i), ++count); } } TEST_F(HostArrayTest, Accessors) { ASSERT_EQ(array_.front(), 1); ASSERT_EQ(array_.back(), 5); } SpFFT-0.9.10/tests/local_tests/test_local_transform.cpp000066400000000000000000000100171357247005600231410ustar00rootroot00000000000000#include #include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/grid.hpp" #include "spfft/transform.hpp" #include "test_util/generate_indices.hpp" #include "test_util/test_check_values.hpp" #include "test_util/test_transform.hpp" #include "util/common_types.hpp" class TestLocalTransform : public TransformTest { protected: TestLocalTransform() : TransformTest(), grid_(dimX_, dimY_, dimZ_, dimX_ * dimY_, std::get<1>(GetParam()), -1) {} auto grid() -> Grid& override { return grid_; } Grid grid_; }; TEST_P(TestLocalTransform, ForwardC2C) { try { std::vector zStickDistribution(comm_size(), 1.0); std::vector xyPlaneDistribution(comm_size(), 1.0); test_forward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(TestLocalTransform, BackwardC2C) { try { std::vector zStickDistribution(comm_size(), 1.0); std::vector xyPlaneDistribution(comm_size(), 1.0); test_backward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(TestLocalTransform, R2C) { try { std::vector xyPlaneDistribution(comm_size(), 1.0); test_r2c(xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } // Show exchange name instead of enum value for test output static auto param_type_names( const ::testing::TestParamInfo< std::tuple>& info) -> std::string { const auto exchType = std::get<0>(info.param); const auto procType = std::get<1>(info.param); std::string name; switch (procType) { case SpfftProcessingUnitType::SPFFT_PU_HOST: { name += "Host"; } break; case SpfftProcessingUnitType::SPFFT_PU_GPU: { name += "GPU"; } break; default: { name += "Host+GPU"; } } name += "Size"; name += std::to_string(std::get<2>(info.param)); name += "x"; name += std::to_string(std::get<3>(info.param)); name += "x"; name += std::to_string(std::get<4>(info.param)); return name; } // instantiate tests with parameters #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #define TEST_PROCESSING_UNITS \ SpfftProcessingUnitType::SPFFT_PU_HOST, SpfftProcessingUnitType::SPFFT_PU_GPU #else #define TEST_PROCESSING_UNITS SpfftProcessingUnitType::SPFFT_PU_HOST #endif INSTANTIATE_TEST_CASE_P(FullTest, TestLocalTransform, ::testing::Combine(::testing::Values(SpfftExchangeType::SPFFT_EXCH_DEFAULT), ::testing::Values(TEST_PROCESSING_UNITS), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(false)), param_type_names); INSTANTIATE_TEST_CASE_P(CenteredIndicesTest, TestLocalTransform, ::testing::Combine(::testing::Values(SpfftExchangeType::SPFFT_EXCH_DEFAULT), ::testing::Values(TEST_PROCESSING_UNITS), ::testing::Values(1, 2, 11, 100), ::testing::Values(1, 2, 11, 100), ::testing::Values(1, 2, 11, 100), ::testing::Values(true)), param_type_names); SpFFT-0.9.10/tests/mpi_tests/000077500000000000000000000000001357247005600157055ustar00rootroot00000000000000SpFFT-0.9.10/tests/mpi_tests/test_multi_transform.cpp000066400000000000000000000065611357247005600227050ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "parameters/parameters.hpp" #include "spfft/spfft.hpp" #include "test_util/generate_indices.hpp" #include "test_util/test_transform.hpp" #include "util/common_types.hpp" TEST(MPIMultiTransformTest, BackwardsForwards) { try { MPICommunicatorHandle comm(MPI_COMM_WORLD); const std::vector zStickDistribution(comm.size(), 1.0); const std::vector xyPlaneDistribution(comm.size(), 1.0); const int dimX = comm.size() * 10; const int dimY = comm.size() * 11; const int dimZ = comm.size() * 12; const int numTransforms = 3; std::mt19937 randGen(42); const auto valueIndicesPerRank = create_value_indices(randGen, zStickDistribution, 0.7, 0.7, dimX, dimY, dimZ, false); const int numLocalXYPlanes = calculate_num_local_xy_planes(comm.rank(), dimZ, xyPlaneDistribution); const auto& localIndices = valueIndicesPerRank[comm.rank()]; const int numValues = localIndices.size() / 3; std::vector>> freqValuesPerTrans( numTransforms, std::vector>(numValues)); std::vector freqValuePtr; for (auto& values : freqValuesPerTrans) { freqValuePtr.push_back(reinterpret_cast(values.data())); } // set frequency values to constant for each transform for (std::size_t i = 0; i < freqValuesPerTrans.size(); ++i) { for (auto& val : freqValuesPerTrans[i]) { val = std::complex(i, i); } } std::vector transforms; // create first transforms transforms.push_back(Grid(dimX, dimY, dimZ, dimX * dimY, numLocalXYPlanes, SPFFT_PU_HOST, -1, comm.get(), SPFFT_EXCH_DEFAULT) .create_transform(SPFFT_PU_HOST, SPFFT_TRANS_C2C, dimX, dimY, dimZ, numLocalXYPlanes, numValues, SPFFT_INDEX_TRIPLETS, localIndices.data())); // clone first transform for (int i = 1; i < numTransforms; ++i) { transforms.push_back(transforms.front().clone()); } std::vector processingUnits(numTransforms, SPFFT_PU_HOST); std::vector scalingTypes(numTransforms, SPFFT_NO_SCALING); // backward multi_transform_backward(numTransforms, transforms.data(), freqValuePtr.data(), processingUnits.data()); // forward multi_transform_forward(numTransforms, transforms.data(), processingUnits.data(), freqValuePtr.data(), scalingTypes.data()); // check all values for (std::size_t i = 0; i < freqValuesPerTrans.size(); ++i) { const auto targetValue = std::complex(i * dimX * dimY * dimZ, i * dimX * dimY * dimZ); for (auto& val : freqValuesPerTrans[i]) { ASSERT_NEAR(targetValue.real(), val.real(), 1e-8); ASSERT_NEAR(targetValue.imag(), val.imag(), 1e-8); } } } catch (const std::exception& e) { std::cout << "ERROR: " << e.what() << std::endl; ASSERT_TRUE(false); } } SpFFT-0.9.10/tests/mpi_tests/test_transform.cpp000066400000000000000000000151301357247005600214630ustar00rootroot00000000000000#include "test_util/test_transform.hpp" #include #include #include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "parameters/parameters.hpp" #include "spfft/grid.hpp" #include "spfft/transform.hpp" #include "test_util/generate_indices.hpp" #include "test_util/test_check_values.hpp" #include "util/common_types.hpp" class MPITransformTest : public TransformTest { protected: MPITransformTest() : TransformTest(), comm_(MPI_COMM_WORLD), grid_(dimX_, dimY_, dimZ_, dimX_ * dimY_, dimZ_, std::get<1>(GetParam()), -1, comm_.get(), std::get<0>(GetParam())) {} auto comm_rank() -> SizeType override { return comm_.rank(); } auto comm_size() -> SizeType override { return comm_.size(); } auto grid() -> Grid& override { return grid_; } MPICommunicatorHandle comm_; Grid grid_; }; TEST_P(MPITransformTest, ForwardUniformDistribution) { try { std::vector zStickDistribution(comm_size(), 1.0); std::vector xyPlaneDistribution(comm_size(), 1.0); test_forward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, BackwardAllOneRank) { try { std::vector zStickDistribution(comm_size(), 0.0); zStickDistribution[0] = 1.0; std::vector xyPlaneDistribution(comm_size(), 0.0); xyPlaneDistribution[0] = 1.0; test_backward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, ForwardAllOneRank) { try { std::vector zStickDistribution(comm_size(), 0.0); zStickDistribution[0] = 1.0; std::vector xyPlaneDistribution(comm_size(), 0.0); xyPlaneDistribution[0] = 1.0; test_forward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, BackwardAllOneRankPerSide) { try { std::vector zStickDistribution(comm_size(), 0.0); zStickDistribution[0] = 1.0; std::vector xyPlaneDistribution(comm_size(), 0.0); xyPlaneDistribution[comm_size() - 1] = 1.0; test_backward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, ForwardAllOneRankPerSide) { try { std::vector zStickDistribution(comm_size(), 0.0); zStickDistribution[0] = 1.0; std::vector xyPlaneDistribution(comm_size(), 0.0); xyPlaneDistribution[comm_size() - 1] = 1.0; test_forward_c2c(zStickDistribution, xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, R2CUniformDistribution) { try { std::vector xyPlaneDistribution(comm_size(), 1.0); test_r2c(xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } TEST_P(MPITransformTest, R2COneRankAllPlanes) { try { std::vector xyPlaneDistribution(comm_size(), 0.0); xyPlaneDistribution[0] = 1.0; test_r2c(xyPlaneDistribution); } catch (const std::exception& e) { std::cout << "ERROR: Rank " << comm_rank() << ", " << e.what() << std::endl; ASSERT_TRUE(false); } } // Show exchange name instead of enum value for test output static auto param_type_names( const ::testing::TestParamInfo< std::tuple>& info) -> std::string { const auto exchType = std::get<0>(info.param); const auto procType = std::get<1>(info.param); std::string name; switch (exchType) { case SpfftExchangeType::SPFFT_EXCH_BUFFERED: { name += "Buffered"; } break; case SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED: { name += "CompactBuffered"; } break; case SpfftExchangeType::SPFFT_EXCH_UNBUFFERED: { name += "Unbuffered"; } break; default: name += "Default"; } switch (procType) { case SpfftProcessingUnitType::SPFFT_PU_HOST: { name += "Host"; } break; case SpfftProcessingUnitType::SPFFT_PU_GPU: { name += "GPU"; } break; default: { name += "Host+GPU"; } } name += "Size"; name += std::to_string(std::get<2>(info.param)); name += "x"; name += std::to_string(std::get<3>(info.param)); name += "x"; name += std::to_string(std::get<4>(info.param)); return name; } // instantiate tests with parameters #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #define TEST_PROCESSING_UNITS \ SpfftProcessingUnitType::SPFFT_PU_HOST, SpfftProcessingUnitType::SPFFT_PU_GPU #else #define TEST_PROCESSING_UNITS SpfftProcessingUnitType::SPFFT_PU_HOST #endif INSTANTIATE_TEST_CASE_P( FullTest, MPITransformTest, ::testing::Combine(::testing::Values(SpfftExchangeType::SPFFT_EXCH_BUFFERED, SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED, SpfftExchangeType::SPFFT_EXCH_UNBUFFERED, SpfftExchangeType::SPFFT_EXCH_DEFAULT), ::testing::Values(TEST_PROCESSING_UNITS), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(1, 2, 11, 12, 13, 100), ::testing::Values(false)), param_type_names); INSTANTIATE_TEST_CASE_P(CenteredIndicesTest, MPITransformTest, ::testing::Combine(::testing::Values(SpfftExchangeType::SPFFT_EXCH_DEFAULT), ::testing::Values(TEST_PROCESSING_UNITS), ::testing::Values(1, 2, 11, 100), ::testing::Values(1, 2, 11, 100), ::testing::Values(1, 2, 11, 100), ::testing::Values(true)), param_type_names); SpFFT-0.9.10/tests/mpi_tests/test_transpose.cpp000066400000000000000000000174711357247005600215000ustar00rootroot00000000000000#include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "parameters/parameters.hpp" #include "transpose/transpose_mpi_buffered_host.hpp" #include "transpose/transpose_mpi_compact_buffered_host.hpp" #include "transpose/transpose_mpi_unbuffered_host.hpp" #include "util/common_types.hpp" using namespace spfft; class TransposeTest : public ::testing::Test { protected: void SetUp() override { comm_ = MPICommunicatorHandle(MPI_COMM_WORLD); SizeType dimX = 2 * comm_.size(); SizeType dimY = 3 * comm_.size(); SizeType dimZ = 4 * comm_.size(); // create memory space array1_ = HostArray>(dimX * dimY * dimZ, std::complex(1.0, 1.0)); array2_ = HostArray>(dimX * dimY * dimZ, std::complex(1.0, 1.0)); fullArray_ = HostArray>(dimX * dimY * dimZ); // plane split between ranks const SizeType numLocalXYPlanes = (dimZ / comm_.size()) + (comm_.rank() == comm_.size() - 1 ? dimZ % comm_.size() : 0); const SizeType localXYPlaneOffset = (dimZ / comm_.size()) * comm_.rank(); // create all indices the same way (random generator must be equally initialized) std::mt19937 sharedRandGen(42); std::uniform_real_distribution dis(0.0, 1.0); std::uniform_int_distribution rankSelector(0, comm_.size() - 1); std::vector indexTriplets; indexTriplets.reserve(dimX * dimY * dimZ); for (int x = 0; x < static_cast(dimX); ++x) { for (int y = 0; y < static_cast(dimY); ++y) { // create sparse z stick distribution if (dis(sharedRandGen) < 0.5 && rankSelector(sharedRandGen) == static_cast(comm_.size())) { for (int z = 0; z < static_cast(dimY); ++z) { indexTriplets.push_back(x); indexTriplets.push_back(y); indexTriplets.push_back(z); } } } } paramPtr_.reset(new Parameters(comm_, SPFFT_TRANS_C2C, dimX, dimY, dimZ, numLocalXYPlanes, indexTriplets.size() / 3, SPFFT_INDEX_TRIPLETS, indexTriplets.data())); // initialize random z-stick data auto fullView = create_3d_view(fullArray_, 0, dimX, dimY, dimZ); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), dimZ); for (SizeType r = 0; r < comm_.size(); ++r) { for (const auto& stickIdx : paramPtr_->z_stick_xy_indices(r)) { const auto x = stickIdx / dimY; const auto y = stickIdx - x * dimY; for (SizeType z = 0; z < freqView.dim_inner(); ++z) { fullView(x, y, z) = std::complex(dis(sharedRandGen), dis(sharedRandGen)); } } } // copy data into sticks SizeType count = 0; for (const auto& stickIdx : paramPtr_->z_stick_xy_indices(comm_.rank())) { const auto x = stickIdx / dimY; const auto y = stickIdx - x * dimY; for (SizeType z = 0; z < freqView.dim_inner(); ++z) { freqView(count, z) = fullView(x, y, z); } ++count; } } MPICommunicatorHandle comm_; std::shared_ptr paramPtr_; HostArray> array1_; HostArray> array2_; HostArray> fullArray_; }; static void check_space_domain(const HostArrayView3D>& realView, const HostArrayView3D>& fullView, const SizeType planeOffset, const SizeType numLocalXYPlanes) { for (SizeType z = 0; z < numLocalXYPlanes; ++z) { for (SizeType x = 0; x < fullView.dim_outer(); ++x) { for (SizeType y = 0; y < fullView.dim_mid(); ++y) { EXPECT_EQ(realView(z, x, y).real(), fullView(x, y, z + planeOffset).real()); EXPECT_EQ(realView(z, x, y).imag(), fullView(x, y, z + planeOffset).imag()); } } } } static void check_freq_domain(const HostArrayView2D>& freqView, const HostArrayView3D>& fullView, HostArrayConstView1D xyIndices) { for (SizeType stickIdx = 0; stickIdx < freqView.dim_outer(); ++stickIdx) { const auto x = xyIndices(stickIdx) / fullView.dim_outer(); const auto y = xyIndices(stickIdx) - x * fullView.dim_outer(); for (SizeType z = 0; z < freqView.dim_inner(); ++z) { EXPECT_EQ(freqView(stickIdx, z).real(), fullView(x, y, z).real()); EXPECT_EQ(freqView(stickIdx, z).imag(), fullView(x, y, z).imag()); } } } TEST_F(TransposeTest, Unbuffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_x(), paramPtr_->dim_y()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); TransposeMPIUnbufferedHost transpose(paramPtr_, comm_, freqXYView, freqView); transpose.backward(); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } TEST_F(TransposeTest, CompactBuffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_x(), paramPtr_->dim_y()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); auto transposeBufferZ = create_1d_view( array2_, 0, paramPtr_->total_num_xy_planes() * paramPtr_->num_z_sticks(comm_.rank())); auto transposeBufferXY = create_1d_view( array1_, 0, paramPtr_->total_num_z_sticks() * paramPtr_->num_xy_planes(comm_.rank())); TransposeMPICompactBufferedHost transpose(paramPtr_, comm_, freqXYView, freqView, transposeBufferXY, transposeBufferZ); transpose.backward(); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } TEST_F(TransposeTest, Buffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_x(), paramPtr_->dim_y()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); auto transposeBufferZ = create_1d_view( array2_, 0, paramPtr_->max_num_z_sticks() * paramPtr_->max_num_xy_planes() * comm_.size()); auto transposeBufferXY = create_1d_view( array1_, 0, paramPtr_->max_num_z_sticks() * paramPtr_->max_num_xy_planes() * comm_.size()); TransposeMPIBufferedHost transpose(paramPtr_, comm_, freqXYView, freqView, transposeBufferXY, transposeBufferZ); transpose.backward(); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } SpFFT-0.9.10/tests/mpi_tests/test_transpose_gpu.cpp000066400000000000000000000255021357247005600223450ustar00rootroot00000000000000#include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "mpi_util/mpi_communicator_handle.hpp" #include "parameters/parameters.hpp" #include "util/common_types.hpp" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "execution/execution_gpu.hpp" #include "memory/gpu_array.hpp" #include "transpose/transpose_mpi_buffered_gpu.hpp" #include "transpose/transpose_mpi_compact_buffered_gpu.hpp" #include "transpose/transpose_mpi_unbuffered_gpu.hpp" using namespace spfft; class TransposeGPUTest : public ::testing::Test { protected: void SetUp() override { comm_ = MPICommunicatorHandle(MPI_COMM_WORLD); SizeType dimX = 2 * comm_.size(); SizeType dimY = 3 * comm_.size(); SizeType dimZ = 4 * comm_.size(); // create memory space array1_ = HostArray>(dimX * dimY * dimZ, std::complex(1.0, 1.0)); array2_ = HostArray>(dimX * dimY * dimZ, std::complex(1.0, 1.0)); fullArray_ = HostArray>(dimX * dimY * dimZ); gpuArray1_ = GPUArray::type>(array1_.size()); gpuArray2_ = GPUArray::type>(array1_.size()); // pinn arrays array1_.pin_memory(); array2_.pin_memory(); // plane split between ranks const SizeType numLocalXYPlanes = (dimZ / comm_.size()) + (comm_.rank() == comm_.size() - 1 ? dimZ % comm_.size() : 0); const SizeType localXYPlaneOffset = (dimZ / comm_.size()) * comm_.rank(); // create all indices the same way (random generator must be equally initialized) std::mt19937 sharedRandGen(42); std::uniform_real_distribution dis(0.0, 1.0); std::uniform_int_distribution rankSelector(0, comm_.size() - 1); std::vector indexTriplets; indexTriplets.reserve(dimX * dimY * dimZ); for (int x = 0; x < static_cast(dimX); ++x) { for (int y = 0; y < static_cast(dimY); ++y) { // create sparse z stick distribution if (dis(sharedRandGen) < 0.5 && rankSelector(sharedRandGen) == comm_.size()) { for (int z = 0; z < static_cast(dimY); ++z) { indexTriplets.push_back(x); indexTriplets.push_back(y); indexTriplets.push_back(z); } } } } paramPtr_.reset(new Parameters(comm_, SPFFT_TRANS_C2C, dimX, dimY, dimZ, numLocalXYPlanes, indexTriplets.size() / 3, SPFFT_INDEX_TRIPLETS, indexTriplets.data())); // initialize random z-stick data auto fullView = create_3d_view(fullArray_, 0, dimX, dimY, dimZ); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), dimZ); for (SizeType r = 0; r < comm_.size(); ++r) { for (const auto& stickIdx : paramPtr_->z_stick_xy_indices(r)) { const auto x = stickIdx / dimY; const auto y = stickIdx - x * dimY; for (SizeType z = 0; z < freqView.dim_inner(); ++z) { fullView(x, y, z) = std::complex(dis(sharedRandGen), dis(sharedRandGen)); } } } // copy data into sticks SizeType count = 0; for (const auto& stickIdx : paramPtr_->z_stick_xy_indices(comm_.rank())) { const auto x = stickIdx / dimY; const auto y = stickIdx - x * dimY; for (SizeType z = 0; z < freqView.dim_inner(); ++z) { freqView(count, z) = fullView(x, y, z); } ++count; } } MPICommunicatorHandle comm_; std::shared_ptr paramPtr_; HostArray> array1_; HostArray> array2_; HostArray> fullArray_; GPUArray::type> gpuArray1_; GPUArray::type> gpuArray2_; }; static void check_space_domain(const HostArrayView3D>& realView, const HostArrayView3D>& fullView, const SizeType planeOffset, const SizeType numLocalXYPlanes) { for (SizeType z = 0; z < numLocalXYPlanes; ++z) { for (SizeType x = 0; x < fullView.dim_outer(); ++x) { for (SizeType y = 0; y < fullView.dim_mid(); ++y) { EXPECT_EQ(realView(z, x, y).real(), fullView(x, y, z + planeOffset).real()); EXPECT_EQ(realView(z, x, y).imag(), fullView(x, y, z + planeOffset).imag()); } } } } static void check_freq_domain(const HostArrayView2D>& freqView, const HostArrayView3D>& fullView, HostArrayConstView1D xyIndices) { for (SizeType stickIdx = 0; stickIdx < freqView.dim_outer(); ++stickIdx) { const auto x = stickIdx / fullView.dim_outer(); const auto y = stickIdx - x * fullView.dim_outer(); for (SizeType z = 0; z < freqView.dim_inner(); ++z) { EXPECT_EQ(freqView(stickIdx, z).real(), fullView(x, y, z).real()); EXPECT_EQ(freqView(stickIdx, z).imag(), fullView(x, y, z).imag()); } } } TEST_F(TransposeGPUTest, Buffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqXYViewGPU = create_3d_view(gpuArray2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto freqViewGPU = create_2d_view(gpuArray1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); GPUStreamHandle stream(false); auto transposeBufferZ = create_1d_view( array2_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferZGPU = create_1d_view( gpuArray2_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferXY = create_1d_view( array1_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferXYGPU = create_1d_view( gpuArray1_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); TransposeMPIBufferedGPU transpose( paramPtr_, comm_, transposeBufferXY, freqXYViewGPU, transposeBufferXYGPU, stream, transposeBufferZ, freqViewGPU, transposeBufferZGPU, stream); copy_to_gpu_async(stream, freqView, freqViewGPU); transpose.backward(); copy_from_gpu_async(stream, freqXYViewGPU, freqXYView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); copy_from_gpu_async(stream, freqViewGPU, freqView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } TEST_F(TransposeGPUTest, CompactBuffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqXYViewGPU = create_3d_view(gpuArray2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto freqViewGPU = create_2d_view(gpuArray1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); GPUStreamHandle stream(false); auto transposeBufferZ = create_1d_view( array2_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferZGPU = create_1d_view( gpuArray2_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferXY = create_1d_view( array1_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); auto transposeBufferXYGPU = create_1d_view( gpuArray1_, 0, comm_.size() * paramPtr_->max_num_xy_planes() * paramPtr_->max_num_z_sticks()); TransposeMPICompactBufferedGPU transpose( paramPtr_, comm_, transposeBufferXY, freqXYViewGPU, transposeBufferXYGPU, stream, transposeBufferZ, freqViewGPU, transposeBufferZGPU, stream); copy_to_gpu_async(stream, freqView, freqViewGPU); transpose.pack_backward(); transpose.backward(); transpose.unpack_backward(); copy_from_gpu_async(stream, freqXYViewGPU, freqXYView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); copy_from_gpu_async(stream, freqViewGPU, freqView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } TEST_F(TransposeGPUTest, Unbuffered) { auto freqXYView = create_3d_view(array2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqXYViewGPU = create_3d_view(gpuArray2_, 0, paramPtr_->num_xy_planes(comm_.rank()), paramPtr_->dim_y(), paramPtr_->dim_x()); auto freqView = create_2d_view(array1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto freqViewGPU = create_2d_view(gpuArray1_, 0, paramPtr_->num_z_sticks(comm_.rank()), paramPtr_->dim_z()); auto fullView = create_3d_view(fullArray_, 0, paramPtr_->dim_x(), paramPtr_->dim_y(), paramPtr_->dim_z()); GPUStreamHandle stream(false); TransposeMPIUnbufferedGPU transpose(paramPtr_, comm_, freqXYView, freqXYViewGPU, stream, freqView, freqViewGPU, stream); copy_to_gpu_async(stream, freqView, freqViewGPU); transpose.backward(); copy_from_gpu_async(stream, freqXYViewGPU, freqXYView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_space_domain(freqXYView, fullView, paramPtr_->xy_plane_offset(comm_.rank()), paramPtr_->num_xy_planes(comm_.rank())); transpose.forward(); copy_from_gpu_async(stream, freqViewGPU, freqView); gpu::check_status(gpu::stream_synchronize(stream.get())); check_freq_domain(freqView, fullView, paramPtr_->z_stick_xy_indices(comm_.rank())); } #endif SpFFT-0.9.10/tests/programs/000077500000000000000000000000001357247005600155305ustar00rootroot00000000000000SpFFT-0.9.10/tests/programs/benchmark.cpp000066400000000000000000000300341357247005600201660ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include "fft/transform_1d_host.hpp" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "spfft/config.h" #include "spfft/spfft.hpp" #include "timing/timing.hpp" #include "util/omp_definitions.hpp" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "gpu_util/gpu_runtime_api.hpp" #include "gpu_util/gpu_transfer.hpp" #include "memory/gpu_array.hpp" #endif #ifdef SPFFT_MPI #include #include "mpi_util/mpi_communicator_handle.hpp" #include "mpi_util/mpi_init_handle.hpp" #endif // external dependencies #include "CLI/CLI.hpp" #include "nlohmann/json.hpp" // #include // for MPI debugging using namespace spfft; void run_benchmark(const SpfftTransformType transformType, const int dimX, const int dimY, const int dimZ, const int numLocalZSticks, const int numLocalXYPlanes, const SpfftProcessingUnitType executionUnit, const SpfftProcessingUnitType targetUnit, const int numThreads, const SpfftExchangeType exchangeType, const std::vector& indices, const int numRepeats, const int numTransforms, double** freqValuesPTR) { std::vector transforms; for (int t = 0; t < numTransforms; ++t) { #ifdef SPFFT_MPI Grid grid(dimX, dimY, dimZ, numLocalZSticks, numLocalXYPlanes, executionUnit, numThreads, MPI_COMM_WORLD, exchangeType); #else Grid grid(dimX, dimY, dimZ, numLocalZSticks, executionUnit, numThreads); #endif auto transform = grid.create_transform( executionUnit, transformType, dimX, dimY, dimZ, numLocalXYPlanes, indices.size() / 3, SpfftIndexFormatType::SPFFT_INDEX_TRIPLETS, indices.data()); transforms.emplace_back(std::move(transform)); } std::vector targetUnits(numTransforms, targetUnit); std::vector scalingTypes(numTransforms, SPFFT_NO_SCALING); // run once for warm cache { HOST_TIMING_SCOPED("Warming") multi_transform_backward(transforms.size(), transforms.data(), freqValuesPTR, targetUnits.data()); multi_transform_forward(transforms.size(), transforms.data(), targetUnits.data(), freqValuesPTR, scalingTypes.data()); } std::string exchName("Compact buffered"); if (exchangeType == SpfftExchangeType::SPFFT_EXCH_BUFFERED) { exchName = "Buffered"; } else if (exchangeType == SpfftExchangeType::SPFFT_EXCH_UNBUFFERED) { exchName = "Unbuffered"; } else if (exchangeType == SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED_FLOAT) { exchName = "Compact buffered float"; } else if (exchangeType == SpfftExchangeType::SPFFT_EXCH_BUFFERED_FLOAT) { exchName = "Buffered float"; } HOST_TIMING_SCOPED(exchName) if (numTransforms == 1) { for (int repeat = 0; repeat < numRepeats; ++repeat) { transforms.front().backward(*freqValuesPTR, targetUnits.front()); transforms.front().forward(targetUnits.front(), *freqValuesPTR, scalingTypes.front()); } } else { for (int repeat = 0; repeat < numRepeats; ++repeat) { multi_transform_backward(transforms.size(), transforms.data(), freqValuesPTR, targetUnits.data()); multi_transform_forward(transforms.size(), transforms.data(), targetUnits.data(), freqValuesPTR, scalingTypes.data()); } } } int main(int argc, char** argv) { #ifdef SPFFT_MPI MPIInitHandle initHandle(argc, argv, true); MPICommunicatorHandle comm(MPI_COMM_WORLD); const SizeType commRank = comm.rank(); const SizeType commSize = comm.size(); #else const SizeType commRank = 0; const SizeType commSize = 1; #endif #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) // set device for multi-gpu nodes int deviceCount = 0; gpu::check_status(gpu::get_device_count(&deviceCount)); if (deviceCount > 1) { gpu::check_status(gpu::set_device(commRank % deviceCount)); } #endif // if(commRank == 0) { // std::cout << "PID = " << getpid() << std::endl; // } // bool waitLoop = commRank == 0; // while(waitLoop) { // sleep(5); // } int numRepeats = 1; int numTransforms = 1; std::string outputFileName; std::string exchName; std::string procName; std::string transformTypeName = "c2c"; double sparsity = 1.0; std::vector dimensions; CLI::App app{"fft test"}; app.add_option("-d", dimensions, "Size of fft grid in each dimension")->required()->expected(3); app.add_option("-r", numRepeats, "Number of repeats")->required(); app.add_option("-o", outputFileName, "Output file name")->required(); app.add_option("-m", numTransforms, "Multiple transform number")->default_val("1"); app.add_option("-s", sparsity, "Sparsity"); app.add_set("-t", transformTypeName, std::set{"c2c", "r2c"}, "Transform type") ->default_val("c2c"); app.add_set("-e", exchName, std::set{"all", "compact", "compactFloat", "buffered", "bufferedFloat", "unbuffered"}, "Exchange type") ->required(); app.add_set("-p", procName, std::set{"cpu", "gpu", "gpu-gpu"}, "Processing unit. With gpu-gpu, device memory is used as input and output.") ->required(); CLI11_PARSE(app, argc, argv); auto transformType = SPFFT_TRANS_C2C; if(transformTypeName == "r2c") { transformType = SPFFT_TRANS_R2C; } const int dimX = dimensions[0]; const int dimY = dimensions[1]; const int dimZ = dimensions[2]; const int dimXFreq = transformType == SPFFT_TRANS_R2C ? dimX / 2 + 1 : dimX; const int dimYFreq = transformType == SPFFT_TRANS_R2C ? dimY / 2 + 1 : dimY; const int dimZFreq = transformType == SPFFT_TRANS_R2C ? dimZ / 2 + 1 : dimZ; const int numThreads = omp_get_max_threads(); const SizeType numLocalXYPlanes = (dimZ / commSize) + (commRank < dimZ % commSize ? 1 : 0); int numLocalZSticks = 0; std::vector xyzIndices; { // std::mt19937 randGen(42); // std::uniform_real_distribution uniformRandDis(0.0, 1.0); // create all global x-y index pairs std::vector> xyIndicesGlobal; xyIndicesGlobal.reserve(dimX * dimY); for (int x = 0; x < dimXFreq * sparsity; ++x) { for (int y = 0; y < (x == 0 ? dimYFreq : dimY); ++y) { xyIndicesGlobal.emplace_back(x, y); } } // distribute z-sticks as evenly as possible numLocalZSticks = (xyIndicesGlobal.size()) / commSize + (commRank < (xyIndicesGlobal.size()) % commSize ? 1 : 0); const int offset = ((xyIndicesGlobal.size()) / commSize) * commRank + std::min(commRank, static_cast(xyIndicesGlobal.size()) % commSize); // assemble index triplets xyzIndices.reserve(numLocalZSticks); for (int i = offset; i < offset + numLocalZSticks; ++i) { for (int z = 0; z < dimZ; ++z) { xyzIndices.push_back(xyIndicesGlobal[i].first); xyzIndices.push_back(xyIndicesGlobal[i].second); xyzIndices.push_back(z); } } } // store full z-sticks values const auto executionUnit = procName == "cpu" ? SpfftProcessingUnitType::SPFFT_PU_HOST : SpfftProcessingUnitType::SPFFT_PU_GPU; const auto targetUnit = procName == "gpu-gpu" ? SpfftProcessingUnitType::SPFFT_PU_GPU : SpfftProcessingUnitType::SPFFT_PU_HOST; std::vector freqValuesPointers(numTransforms); std::vector>> freqValues; for (int t = 0; t < numTransforms; ++t) freqValues.emplace_back(xyzIndices.size() / 3); #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) std::vector> freqValuesGPU; for (int t = 0; t < numTransforms; ++t) freqValuesGPU.emplace_back(xyzIndices.size() / 3); for (int t = 0; t < numTransforms; ++t) { freqValuesPointers[t] = procName == "gpu-gpu" ? reinterpret_cast(freqValuesGPU[t].data()) : reinterpret_cast(freqValues[t].data()); } #else for (int t = 0; t < numTransforms; ++t) { freqValuesPointers[t] = reinterpret_cast(freqValues[t].data()); } #endif #ifdef SPFFT_GPU_DIRECT const bool gpuDirectEnabled = true; #else const bool gpuDirectEnabled = false; #endif if (commRank == 0) { std::cout << "Num MPI ranks: " << commSize << std::endl; std::cout << "Grid size: " << dimX << ", " << dimY << ", " << dimZ << std::endl; std::cout << "Transform type: " << transformTypeName << std::endl; std::cout << "Sparsity: " << sparsity << std::endl; std::cout << "Proc: " << procName << std::endl; std::cout << "GPU Direct: " << (gpuDirectEnabled ? "Enabled" : "Disabled") << std::endl; } if (exchName == "all") { run_benchmark(transformType, dimX, dimY, dimZ, numLocalZSticks, numLocalXYPlanes, executionUnit, targetUnit, numThreads, SpfftExchangeType::SPFFT_EXCH_BUFFERED, xyzIndices, numRepeats, numTransforms, freqValuesPointers.data()); run_benchmark(transformType, dimX, dimY, dimZ, numLocalZSticks, numLocalXYPlanes, executionUnit, targetUnit, numThreads, SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED, xyzIndices, numRepeats, numTransforms, freqValuesPointers.data()); run_benchmark(transformType, dimX, dimY, dimZ, numLocalZSticks, numLocalXYPlanes, executionUnit, targetUnit, numThreads, SpfftExchangeType::SPFFT_EXCH_UNBUFFERED, xyzIndices, numRepeats, numTransforms, freqValuesPointers.data()); } else { auto exchangeType = SpfftExchangeType::SPFFT_EXCH_DEFAULT; if (exchName == "compact") { exchangeType = SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED; } else if (exchName == "compactFloat") { exchangeType = SpfftExchangeType::SPFFT_EXCH_COMPACT_BUFFERED_FLOAT; } else if (exchName == "buffered") { exchangeType = SpfftExchangeType::SPFFT_EXCH_BUFFERED; } else if (exchName == "bufferedFloat") { exchangeType = SpfftExchangeType::SPFFT_EXCH_BUFFERED_FLOAT; } else if (exchName == "unbuffered") { exchangeType = SpfftExchangeType::SPFFT_EXCH_UNBUFFERED; } run_benchmark(transformType, dimX, dimY, dimZ, numLocalZSticks, numLocalXYPlanes, executionUnit, targetUnit, numThreads, exchangeType, xyzIndices, numRepeats, numTransforms, freqValuesPointers.data()); } if (commRank == 0) { auto timingResults = ::spfft::timing::GlobalTimer.process(); std::cout << timingResults.print({::rt_graph::Stat::Count, ::rt_graph::Stat::Total, ::rt_graph::Stat::Percentage, ::rt_graph::Stat::ParentPercentage, ::rt_graph::Stat::Median, ::rt_graph::Stat::Min, ::rt_graph::Stat::Max}) << std::endl; if (!outputFileName.empty()) { nlohmann::json j; const std::time_t t = std::time(nullptr); std::string time(std::ctime(&t)); time.pop_back(); j["timings"] =nlohmann::json::parse(timingResults.json()); const bool data_on_gpu = procName == "gpu-gpu"; j["parameters"] = {{"proc", procName}, {"data_on_gpu", data_on_gpu}, {"gpu_direct", gpuDirectEnabled}, {"num_ranks", commSize}, {"num_threads", numThreads}, {"dim_x", dimX}, {"dim_y", dimY}, {"dim_z", dimZ}, {"exchange_type", exchName}, {"num_repeats", numRepeats}, {"transform_type", transformTypeName}, {"time", time}}; std::ofstream file(outputFileName); file << std::setw(2) << j; file.close(); } } return 0; } SpFFT-0.9.10/tests/run_local_tests.cpp000066400000000000000000000002031357247005600175750ustar00rootroot00000000000000#include "gtest/gtest.h" int main(int argc, char *argv[]) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } SpFFT-0.9.10/tests/run_mpi_tests.cpp000066400000000000000000000015371357247005600173030ustar00rootroot00000000000000#include #include "gtest/gtest.h" #include "gtest_mpi/gtest_mpi.hpp" int main(int argc, char* argv[]) { // Initialize MPI before any call to gtest_mpi MPI_Init(&argc, &argv); // Intialize google test ::testing::InitGoogleTest(&argc, argv); // Add a test envirnment, which will initialize a test communicator // (a duplicate of MPI_COMM_WORLD) ::testing::AddGlobalTestEnvironment(new gtest_mpi::MPITestEnvironment()); auto& test_listeners = ::testing::UnitTest::GetInstance()->listeners(); // Remove default listener and replace with the custom MPI listener delete test_listeners.Release(test_listeners.default_result_printer()); test_listeners.Append(new gtest_mpi::PrettyMPIUnitTestResultPrinter()); // run tests auto exit_code = RUN_ALL_TESTS(); // Finalize MPI before exiting MPI_Finalize(); return exit_code; } SpFFT-0.9.10/tests/test_util/000077500000000000000000000000001357247005600157125ustar00rootroot00000000000000SpFFT-0.9.10/tests/test_util/generate_indices.hpp000066400000000000000000000135761357247005600217270ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_GENERATE_INDICES_HPP #define SPFFT_GENERATE_INDICES_HPP #include #include #include #include "spfft/config.h" namespace spfft { // creates randomly distributed indices for all ranks according to the input distributions template auto create_value_indices(T& sharedRandGen, const std::vector& zStickDistribution, const double totalZStickFraction, const double zStickFillFraction, const int dimX, const int dimY, const int dimZ, const bool hermitianSymmetry) -> std::vector> { std::uniform_real_distribution uniformRandDis(0.0, 1.0); std::discrete_distribution rankSelectDis(zStickDistribution.begin(), zStickDistribution.end()); const double zStickFractionSum = std::accumulate(zStickDistribution.begin(), zStickDistribution.end(), 0.0); std::vector>> xyIndicesPerRank(zStickDistribution.size()); const int dimXFreq = hermitianSymmetry ? dimX / 2 + 1 : dimX; const int dimYFreq = hermitianSymmetry ? dimY / 2 + 1 : dimY; for (int x = 0; x < dimXFreq; ++x) { for (int y = 0; y < dimY; ++y) { if (!(x == 0 && y >= dimYFreq) && uniformRandDis(sharedRandGen) < totalZStickFraction) { // use full hermitian symmetry on x = 0 plane if (!hermitianSymmetry || x != 0 || y < dimYFreq) { const auto selectedRank = rankSelectDis(sharedRandGen); xyIndicesPerRank[selectedRank].emplace_back(std::make_pair(x, y)); } } } } const int dimZFreq = hermitianSymmetry ? dimZ / 2 + 1 : dimZ; std::vector> valueIndices(zStickDistribution.size()); auto valueIndicesIt = valueIndices.begin(); for (const auto& xyIndices : xyIndicesPerRank) { for (const auto& xyIndex : xyIndices) { for (int z = 0; z < dimZ; ++z) { // only add half x=0, y=0 stick if hermitian symmetry is used if (!(hermitianSymmetry && xyIndex.first == 0 && xyIndex.second == 0 && z >= dimZFreq) && uniformRandDis(sharedRandGen) < zStickFillFraction) { valueIndicesIt->emplace_back(xyIndex.first); valueIndicesIt->emplace_back(xyIndex.second); valueIndicesIt->emplace_back(z); } } } ++valueIndicesIt; } return valueIndices; } inline auto center_indices(const int dimX, const int dimY, const int dimZ, std::vector>& indicesPerRank) -> void { const int positiveSizeX = dimX / 2 + 1; const int positiveSizeY = dimY / 2 + 1; const int positiveSizeZ = dimZ / 2 + 1; for (auto& rankIndices : indicesPerRank) { for (std::size_t i = 0; i < rankIndices.size(); i += 3) { if (rankIndices[i] >= positiveSizeX) rankIndices[i] -= dimX; if (rankIndices[i + 1] >= positiveSizeY) rankIndices[i + 1] -= dimY; if (rankIndices[i + 2] >= positiveSizeZ) rankIndices[i + 2] -= dimZ; } } } // assigns a number of xy planes to the local rank according to the xy plane distribution inline auto calculate_num_local_xy_planes(const int rank, const int dimZ, const std::vector& planeRankDistribution) -> int { const double planeDistriSum = std::accumulate(planeRankDistribution.begin(), planeRankDistribution.end(), 0.0); std::vector numXYPlanesPerRank(planeRankDistribution.size()); for (std::size_t i = 0; i < planeRankDistribution.size(); ++i) { numXYPlanesPerRank[i] = planeRankDistribution[i] / planeDistriSum * dimZ; } int numMissingPlanes = dimZ - std::accumulate(numXYPlanesPerRank.begin(), numXYPlanesPerRank.end(), 0); for (auto& val : numXYPlanesPerRank) { // add missing planes to rank with non-zero number if (val > 0 && numMissingPlanes > 0) { val += numMissingPlanes; numMissingPlanes = 0; break; } // substract extra planes if (numMissingPlanes < 0) { val -= std::min(val, -numMissingPlanes); numMissingPlanes += val; if (numMissingPlanes >= 0) { numMissingPlanes = 0; break; } } } // if all ranks have 0 planes, some planes have to be assigned somewhere if (numMissingPlanes > 0) { numXYPlanesPerRank[0] = numMissingPlanes; } return numXYPlanesPerRank[rank]; } } // namespace spfft #endif SpFFT-0.9.10/tests/test_util/test_check_values.hpp000066400000000000000000000072301357247005600221200ustar00rootroot00000000000000/* * Copyright (c) 2019 ETH Zurich, Simon Frasch * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef SPFFT_TEST_CHECK_VALUES_HPP #define SPFFT_TEST_CHECK_VALUES_HPP #include #include #include #include "gtest/gtest.h" #include "memory/host_array_view.hpp" #include "spfft/config.h" namespace spfft { inline void check_c2c_space_domain(const HostArrayView3D>& realView, const HostArrayView3D>& fftwView, const SizeType planeOffset, const SizeType numLocalXYPlanes) { for (SizeType z = 0; z < numLocalXYPlanes; ++z) { for (SizeType x = 0; x < fftwView.dim_outer(); ++x) { for (SizeType y = 0; y < fftwView.dim_mid(); ++y) { ASSERT_NEAR(realView(z, y, x).real(), fftwView(x, y, z + planeOffset).real(), 1e-6); ASSERT_NEAR(realView(z, y, x).imag(), fftwView(x, y, z + planeOffset).imag(), 1e-6); } } } } inline void check_r2c_space_domain(const HostArrayView3D& realView, const HostArrayView3D>& fftwView, const SizeType planeOffset, const SizeType numLocalXYPlanes) { for (SizeType z = 0; z < numLocalXYPlanes; ++z) { for (SizeType x = 0; x < fftwView.dim_outer(); ++x) { for (SizeType y = 0; y < fftwView.dim_mid(); ++y) { ASSERT_NEAR(realView(z, y, x), fftwView(x, y, z + planeOffset).real(), 1e-6); } } } } inline void check_freq_domain(const std::vector>& freqValues, const HostArrayView3D>& fftwView, const std::vector& indices) { assert(indices.size() == freqValues.size() * 3); for (SizeType i = 0; i < freqValues.size(); ++i) { int x = indices[i * 3]; int y = indices[i * 3 + 1]; int z = indices[i * 3 + 2]; if (x < 0) x = fftwView.dim_outer() + x; if (y < 0) y = fftwView.dim_mid() + y; if (z < 0) z = fftwView.dim_inner() + z; ASSERT_NEAR(freqValues[i].real(), fftwView(x, y, z).real(), 1e-6); ASSERT_NEAR(freqValues[i].imag(), fftwView(x, y, z).imag(), 1e-6); } } } // namespace spfft #endif SpFFT-0.9.10/tests/test_util/test_transform.hpp000066400000000000000000000257561357247005600215140ustar00rootroot00000000000000#ifndef SPFFT_TEST_TRANSFORM_HPP #define SPFFT_TEST_TRANSFORM_HPP #include #include #include #include #include #include #include #include "gtest/gtest.h" #include "memory/array_view_utility.hpp" #include "memory/host_array.hpp" #include "memory/host_array_view.hpp" #include "parameters/parameters.hpp" #include "spfft/grid.hpp" #include "spfft/transform.hpp" #include "test_util/generate_indices.hpp" #include "test_util/test_check_values.hpp" #include "util/common_types.hpp" #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) #include "gpu_util/gpu_fft_api.hpp" #include "gpu_util/gpu_transfer.hpp" #include "memory/gpu_array.hpp" #endif using namespace spfft; class TransformTest : public ::testing::TestWithParam< std::tuple> { protected: TransformTest() : dimX_(std::get<2>(GetParam())), dimY_(std::get<3>(GetParam())), dimZ_(std::get<4>(GetParam())), fftwArray_(dimX_ * dimY_ * dimZ_), fftwView_(create_3d_view(fftwArray_, 0, dimX_, dimY_, dimZ_)), centeredIndices_(std::get<5>(GetParam())) { // initialize ffw plans fftwPlanBackward_ = fftw_plan_dft_3d(dimX_, dimY_, dimZ_, (fftw_complex*)fftwArray_.data(), (fftw_complex*)fftwArray_.data(), FFTW_BACKWARD, FFTW_ESTIMATE); fftwPlanForward_ = fftw_plan_dft_3d(dimX_, dimY_, dimZ_, (fftw_complex*)fftwArray_.data(), (fftw_complex*)fftwArray_.data(), FFTW_FORWARD, FFTW_ESTIMATE); } inline auto test_backward_c2c(const std::vector& zStickDistribution, const std::vector& xyPlaneDistribution) -> void; inline auto test_forward_c2c(const std::vector& zStickDistribution, const std::vector& xyPlaneDistribution) -> void; inline auto test_r2c(const std::vector& xyPlaneDistribution) -> void; virtual auto comm_rank() -> SizeType { return 0; } virtual auto comm_size() -> SizeType { return 1; } virtual auto grid() -> Grid& = 0; ~TransformTest() override { if (fftwPlanBackward_) fftw_destroy_plan(fftwPlanBackward_); if (fftwPlanForward_) fftw_destroy_plan(fftwPlanForward_); fftwPlanBackward_ = nullptr; fftwPlanForward_ = nullptr; } int dimX_, dimY_, dimZ_; HostArray> fftwArray_; HostArrayView3D> fftwView_; fftw_plan fftwPlanBackward_ = nullptr; fftw_plan fftwPlanForward_ = nullptr; bool centeredIndices_; }; auto TransformTest::test_backward_c2c(const std::vector& zStickDistribution, const std::vector& xyPlaneDistribution) -> void { std::mt19937 randGen(42); std::uniform_real_distribution uniformRandDis(0.0, 1.0); auto valueIndicesPerRank = create_value_indices(randGen, zStickDistribution, 0.7, 0.7, dimX_, dimY_, dimZ_, false); const int numLocalXYPlanes = calculate_num_local_xy_planes(comm_rank(), dimZ_, xyPlaneDistribution); // assign values to fftw input for (const auto& valueIndices : valueIndicesPerRank) { for (std::size_t i = 0; i < valueIndices.size(); i += 3) { fftwView_(valueIndices[i], valueIndices[i + 1], valueIndices[i + 2]) = std::complex(uniformRandDis(randGen), uniformRandDis(randGen)); } } // extract local rank values std::vector> values(valueIndicesPerRank[comm_rank()].size() / 3); for (std::size_t i = 0; i < values.size(); ++i) { const auto x = valueIndicesPerRank[comm_rank()][i * 3]; const auto y = valueIndicesPerRank[comm_rank()][i * 3 + 1]; const auto z = valueIndicesPerRank[comm_rank()][i * 3 + 2]; values[i] = fftwView_(x, y, z); } if (centeredIndices_) { center_indices(dimX_, dimY_, dimZ_, valueIndicesPerRank); } auto transform = grid().create_transform( std::get<1>(GetParam()), SpfftTransformType::SPFFT_TRANS_C2C, dimX_, dimY_, dimZ_, numLocalXYPlanes, values.size(), SpfftIndexFormatType::SPFFT_INDEX_TRIPLETS, valueIndicesPerRank[comm_rank()].data()); HostArrayView3D> realView( reinterpret_cast*>( transform.space_domain_data(SpfftProcessingUnitType::SPFFT_PU_HOST)), numLocalXYPlanes, dimY_, dimX_, false); fftw_execute(fftwPlanBackward_); #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) if (std::get<1>(GetParam()) == SpfftProcessingUnitType::SPFFT_PU_GPU) { // copy frequency values to GPU GPUArray::type> valuesGPU(values.size()); copy_to_gpu(values, valuesGPU); // transform transform.backward(reinterpret_cast(valuesGPU.data()), SpfftProcessingUnitType::SPFFT_PU_GPU); // run twice to ensure memory is zeroed correctly transform.backward(reinterpret_cast(valuesGPU.data()), SpfftProcessingUnitType::SPFFT_PU_GPU); // use transform buffer to copy values GPUArrayView3D::type> realViewGPU( reinterpret_cast::type*>( transform.space_domain_data(SpfftProcessingUnitType::SPFFT_PU_GPU)), numLocalXYPlanes, dimY_, dimX_, false); copy_from_gpu(realViewGPU, realView); } #endif if (std::get<1>(GetParam()) == SpfftProcessingUnitType::SPFFT_PU_HOST) { transform.backward(reinterpret_cast(values.data()), SpfftProcessingUnitType::SPFFT_PU_HOST); // run twice to ensure memory is zeroed correctly transform.backward(reinterpret_cast(values.data()), SpfftProcessingUnitType::SPFFT_PU_HOST); } check_c2c_space_domain(realView, fftwView_, transform.local_z_offset(), numLocalXYPlanes); } auto TransformTest::test_forward_c2c(const std::vector& zStickDistribution, const std::vector& xyPlaneDistribution) -> void { std::mt19937 randGen(42); std::uniform_real_distribution uniformRandDis(0.0, 1.0); auto valueIndicesPerRank = create_value_indices(randGen, zStickDistribution, 0.7, 0.7, dimX_, dimY_, dimZ_, false); const int numLocalXYPlanes = calculate_num_local_xy_planes(comm_rank(), dimZ_, xyPlaneDistribution); // assign values to fftw input for (const auto& valueIndices : valueIndicesPerRank) { for (std::size_t i = 0; i < valueIndices.size(); i += 3) { fftwView_(valueIndices[i], valueIndices[i + 1], valueIndices[i + 2]) = std::complex(uniformRandDis(randGen), uniformRandDis(randGen)); } } std::vector> freqValues(valueIndicesPerRank[comm_rank()].size() / 3); if (centeredIndices_) { center_indices(dimX_, dimY_, dimZ_, valueIndicesPerRank); } auto transform = grid().create_transform( std::get<1>(GetParam()), SpfftTransformType::SPFFT_TRANS_C2C, dimX_, dimY_, dimZ_, numLocalXYPlanes, freqValues.size(), SpfftIndexFormatType::SPFFT_INDEX_TRIPLETS, valueIndicesPerRank[comm_rank()].data()); HostArrayView3D> realView( reinterpret_cast*>( transform.space_domain_data(SpfftProcessingUnitType::SPFFT_PU_HOST)), numLocalXYPlanes, dimY_, dimX_, false); fftw_execute(fftwPlanBackward_); // copy space domain values from fftw buffer const auto zOffset = transform.local_z_offset(); for (int z = 0; z < numLocalXYPlanes; ++z) { for (int y = 0; y < dimY_; ++y) { for (int x = 0; x < dimX_; ++x) { realView(z, y, x) = fftwView_(x, y, z + zOffset); } } } fftw_execute(fftwPlanForward_); #if defined(SPFFT_CUDA) || defined(SPFFT_ROCM) if (std::get<1>(GetParam()) == SpfftProcessingUnitType::SPFFT_PU_GPU) { // use transform buffer to copy values GPUArrayView3D::type> realViewGPU( reinterpret_cast::type*>( transform.space_domain_data(SpfftProcessingUnitType::SPFFT_PU_GPU)), numLocalXYPlanes, dimY_, dimX_, false); copy_to_gpu(realView, realViewGPU); GPUArray::type> freqValuesGPU(freqValues.size()); transform.forward(SpfftProcessingUnitType::SPFFT_PU_GPU, reinterpret_cast(freqValuesGPU.data())); copy_from_gpu(freqValuesGPU, freqValues); } #endif if (std::get<1>(GetParam()) == SpfftProcessingUnitType::SPFFT_PU_HOST) { transform.forward(SpfftProcessingUnitType::SPFFT_PU_HOST, reinterpret_cast(freqValues.data())); } check_freq_domain(freqValues, fftwView_, valueIndicesPerRank[comm_rank()]); } auto TransformTest::test_r2c(const std::vector& xyPlaneDistribution) -> void { std::mt19937 randGen(42); std::uniform_real_distribution uniformRandDis(0.0, 1.0); // create full set of global z-sticks (up to dimX_ / 2 + 1, due to symmetry) std::vector zStickDistribution(xyPlaneDistribution.size(), 1.0); auto valueIndicesPerRank = create_value_indices(randGen, zStickDistribution, 1.0, 1.0, dimX_, dimY_, dimZ_, true); const int numLocalXYPlanes = calculate_num_local_xy_planes(comm_rank(), dimZ_, xyPlaneDistribution); // assign values to fftw input for (const auto& valueIndices : valueIndicesPerRank) { for (std::size_t i = 0; i < valueIndices.size(); i += 3) { fftwView_(valueIndices[i], valueIndices[i + 1], valueIndices[i + 2]) = std::complex(uniformRandDis(randGen), 0.0); } } std::vector> freqValues(valueIndicesPerRank[comm_rank()].size() / 3); if (centeredIndices_) { center_indices(dimX_, dimY_, dimZ_, valueIndicesPerRank); } auto transform = grid().create_transform( std::get<1>(GetParam()), SpfftTransformType::SPFFT_TRANS_R2C, dimX_, dimY_, dimZ_, numLocalXYPlanes, freqValues.size(), SpfftIndexFormatType::SPFFT_INDEX_TRIPLETS, valueIndicesPerRank[comm_rank()].data()); HostArrayView3D realView( transform.space_domain_data(SpfftProcessingUnitType::SPFFT_PU_HOST), numLocalXYPlanes, dimY_, dimX_, false); // copy space domain values from fftw buffer const auto zOffset = transform.local_z_offset(); for (int z = 0; z < numLocalXYPlanes; ++z) { for (int y = 0; y < dimY_; ++y) { for (int x = 0; x < dimX_; ++x) { realView(z, y, x) = fftwView_(x, y, z + zOffset).real(); } } } // check forward transform.forward(SpfftProcessingUnitType::SPFFT_PU_HOST, reinterpret_cast(freqValues.data())); fftw_execute(fftwPlanForward_); check_freq_domain(freqValues, fftwView_, valueIndicesPerRank[comm_rank()]); // check backward transform.backward(reinterpret_cast(freqValues.data()), SpfftProcessingUnitType::SPFFT_PU_HOST); fftw_execute(fftwPlanBackward_); check_r2c_space_domain(realView, fftwView_, transform.local_z_offset(), numLocalXYPlanes); } #endif