Skip to content

Commit 090dcee

Browse files
jancieskojanciesko
committed
Move Comm into a file
Add force module that uses KRS
1 parent c051efb commit 090dcee

28 files changed

+858
-241
lines changed

‎input/in.lj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ atom_style atomic
55

66
newton off
77
lattice fcc 0.8442
8-
region box block 0 80 0 80 0 80
8+
region box block 0 1 0 1 0 1
99
create_box 1 box
1010
create_atoms 1 box
1111
mass 1 2.0
@@ -20,4 +20,4 @@ neigh_modify every 20 one 50
2020
fix 1 all nve
2121
thermo 10
2222

23-
run 100
23+
run 2

‎src/CMakeLists.txt

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,6 @@ if(ENABLE_MPI)
1515
target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_MPI)
1616
endif()
1717

18-
# Select a default set of options. We can export this as CMake options later
19-
if (ENABLE_KOKKOS_REMOTE_SPACES)
20-
target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
21-
target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_SCALAR)
22-
#target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO)
23-
target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_GLOBAL)
24-
endif()
25-
2618
target_include_directories(ExaMiniMD PRIVATE ${Kokkos_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${SUBDIRECTORIES})
2719
target_link_libraries(ExaMiniMD PRIVATE $<$<BOOL:${ENABLE_MPI}>:MPI::MPI_CXX> Kokkos::kokkos $<$<BOOL:${ENABLE_KOKKOS_REMOTE_SPACES}>:Kokkos::kokkosremotespaces>)
2820

‎src/binning_types/binning_kksort.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ namespace {
7171
void BinningKKSort::create_binning(T_X_FLOAT dx_in, T_X_FLOAT dy_in, T_X_FLOAT dz_in, int halo_depth, bool do_local, bool do_ghost, bool sort) {
7272
if(do_local||do_ghost) {
7373
nhalo = halo_depth;
74-
std::pair<T_INT,T_INT> range(do_local?0:system->N_local,
74+
Kokkos::pair<T_INT,T_INT> range(do_local?0:system->N_local,
7575
do_ghost?system->N_local+system->N_ghost:system->N_local);
7676

7777
nbinx = T_INT(system->sub_domain_x/dx_in);

‎src/comm_lib.cpp

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
//************************************************************************
2+
// ExaMiniMD v. 1.0
3+
// Copyright (2018) National Technology & Engineering Solutions of Sandia,
4+
// LLC (NTESS).
5+
//
6+
// Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
7+
// retains certain rights in this software.
8+
//
9+
// ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
10+
// use in source and binary forms, with or without modification, are
11+
// permitted provided that the following conditions are met:
12+
//
13+
// 1. Redistributions of source code must retain the above copyright notice,
14+
// this list of conditions and the following disclaimer.
15+
//
16+
// 2. Redistributions in binary form must reproduce the above copyright notice,
17+
// this list of conditions and the following disclaimer in the documentation
18+
// and/or other materials provided with the distribution.
19+
//
20+
// 3. Neither the name of the Corporation nor the names of the contributors
21+
// may be used to endorse or promote products derived from this software
22+
// without specific prior written permission.
23+
//
24+
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
25+
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
26+
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27+
// IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
28+
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
29+
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
30+
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31+
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
32+
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
33+
// IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34+
// POSSIBILITY OF SUCH DAMAGE.
35+
//
36+
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
37+
//************************************************************************
38+
39+
#include <comm_lib.h>
40+
#include <assert.h>
41+
42+
#if EXAMINIMD_ENABLE_MPI
43+
#include <mpi.h>
44+
#endif
45+
46+
#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
47+
#include <Kokkos_RemoteSpaces.hpp>
48+
#endif
49+
50+
void comm_lib_init(int argc, char* argv[]) {
51+
#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
52+
int mpi_thread_level_available;
53+
int mpi_thread_level_required = MPI_THREAD_MULTIPLE;
54+
55+
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL
56+
mpi_thread_level_required = MPI_THREAD_SINGLE;
57+
#endif
58+
59+
MPI_Init_thread(&argc, &argv, mpi_thread_level_required,
60+
&mpi_thread_level_available);
61+
assert(mpi_thread_level_available >= mpi_thread_level_required);
62+
63+
#ifdef KRS_ENABLE_SHMEMSPACE
64+
shmem_init_thread(mpi_thread_level_required, &mpi_thread_level_available);
65+
assert(mpi_thread_level_available >= mpi_thread_level_required);
66+
#endif
67+
68+
#ifdef KRS_ENABLE_NVSHMEMSPACE
69+
MPI_Comm mpi_comm;
70+
nvshmemx_init_attr_t attr;
71+
mpi_comm = MPI_COMM_WORLD;
72+
attr.mpi_comm = &mpi_comm;
73+
nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr);
74+
#endif
75+
}
76+
77+
void comm_lib_finalize() {
78+
#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
79+
#ifdef KRS_ENABLE_SHMEMSPACE
80+
shmem_finalize();
81+
#endif
82+
#ifdef KRS_ENABLE_NVSHMEMSPACE
83+
nvshmem_finalize();
84+
#endif
85+
MPI_Finalize();
86+
#endif
87+
#endif
88+
}

‎src/comm_lib.h

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//************************************************************************
2+
// ExaMiniMD v. 1.0
3+
// Copyright (2018) National Technology & Engineering Solutions of Sandia,
4+
// LLC (NTESS).
5+
//
6+
// Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
7+
// retains certain rights in this software.
8+
//
9+
// ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
10+
// use in source and binary forms, with or without modification, are
11+
// permitted provided that the following conditions are met:
12+
//
13+
// 1. Redistributions of source code must retain the above copyright notice,
14+
// this list of conditions and the following disclaimer.
15+
//
16+
// 2. Redistributions in binary form must reproduce the above copyright notice,
17+
// this list of conditions and the following disclaimer in the documentation
18+
// and/or other materials provided with the distribution.
19+
//
20+
// 3. Neither the name of the Corporation nor the names of the contributors
21+
// may be used to endorse or promote products derived from this software
22+
// without specific prior written permission.
23+
//
24+
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
25+
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
26+
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27+
// IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
28+
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
29+
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
30+
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31+
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
32+
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
33+
// IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34+
// POSSIBILITY OF SUCH DAMAGE.
35+
//
36+
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
37+
//************************************************************************
38+
39+
#pragma once
40+
41+
#ifndef COMM_INIT_H
42+
#define COMM_INIT_H
43+
44+
void comm_lib_init(int argc, char* argv[]);
45+
void comm_lib_finalize();
46+
47+
#endif

‎src/comm_types/CMakeLists.txt

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
11
FILE(GLOB SRCS *.cpp)
22
target_sources(ExaMiniMD PRIVATE ${SRCS})
3-
4-
if (!ENABLE_MPI AND !ENABLE_KOKKOS_REMOTE_SPACES)
5-
# Skip MPI module
6-
list(FILTER SRCS EXCLUDE REGEX ".*comm_mpi\\.cpp$")
7-
endif()
8-
93
target_sources(ExaMiniMD PRIVATE ${SRCS})

‎src/comm_types/comm_mpi.cpp

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
3737
//************************************************************************
3838

39+
#include <string>
40+
3941
#if defined(EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
4042
#include<comm_mpi.h>
4143

@@ -197,14 +199,14 @@ void CommMPI::exchange() {
197199
s = *system;
198200
N_local = system->N_local;
199201
N_ghost = 0;
200-
//printf("System A: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
202+
printf("System A: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
201203
Kokkos::parallel_for("CommMPI::exchange_self",
202204
Kokkos::RangePolicy<TagExchangeSelf, Kokkos::IndexType<T_INT> >(0,N_local), *this);
203205

204206
T_INT N_total_recv = 0;
205207
T_INT N_total_send = 0;
206208

207-
//printf("System B: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
209+
printf("System B: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
208210
for(phase = 0; phase < 6; phase ++) {
209211
proc_num_send[phase] = 0;
210212
proc_num_recv[phase] = 0;
@@ -390,7 +392,8 @@ void CommMPI::exchange_halo() {
390392
};
391393

392394
void CommMPI::update_halo() {
393-
#ifndef SHMEMTESTS_USE_HALO
395+
396+
#if !defined(SHMEMTESTS_USE_HALO) && defined(EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
394397
return;
395398
#else
396399
Kokkos::Profiling::pushRegion("Comm::update_halo");
@@ -478,7 +481,13 @@ void CommMPI::update_force() {
478481
Kokkos::Profiling::popRegion();
479482
};
480483

481-
const char* CommMPI::name() { return "CommMPI"; }
484+
const char* CommMPI::name() {
485+
comm_name = std::string("CommMPI");
486+
#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
487+
comm_name += "Distrib";
488+
#endif
489+
return comm_name.c_str();
490+
}
482491

483492
int CommMPI::process_rank() { return proc_rank; }
484493
int CommMPI::num_processes() { return proc_size; }

‎src/comm_types/comm_mpi.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ class CommMPI: public Comm {
7070
System s;
7171

7272
// Owned Variables
73-
7473
int phase; // Communication Phase
7574
int proc_neighbors_recv[6]; // Neighbor for each phase
7675
int proc_neighbors_send[6]; // Neighbor for each phase
@@ -81,6 +80,8 @@ class CommMPI: public Comm {
8180
int proc_rank; // My Process rank
8281
int proc_size; // Number of processes
8382

83+
std::string comm_name;
84+
8485
T_INT num_ghost[6];
8586
T_INT ghost_offsets[6];
8687

@@ -503,7 +504,9 @@ class CommMPI: public Comm {
503504
KOKKOS_INLINE_FUNCTION
504505
void operator() (const TagCreateGlobalIndecies,
505506
const T_INT& i) const {
507+
#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
506508
s.global_index(i) = N_MAX_MASK * proc_rank + i;
509+
#endif
507510
}
508511

509512
const char* name();

‎src/comm_types/comm_serial.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@
5252
}
5353
#endif
5454

55-
5655
#if !defined(MODULES_OPTION_CHECK) && !defined(COMM_MODULES_INSTANTIATION)
5756
#ifndef COMM_SERIAL_H
5857
#define COMM_SERIAL_H

‎src/examinimd.cpp

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,6 @@ void ExaMiniMD::init(int argc, char* argv[]) {
8383
#undef FORCE_MODULES_INSTANTIATION
8484
else comm->error("Invalid ForceType");
8585
for(int line = 0; line < input->force_coeff_lines.extent(0); line++) {
86-
//input->input_data.print_line(input->force_coeff_lines(line));
87-
//printf("init_coeff: %i %i\n",line,input->input_data.words_in_line(input->force_coeff_lines(line)));
8886
force->init_coeff(input->input_data.words_in_line(input->force_coeff_lines(line)),
8987
input->input_data.words[input->force_coeff_lines(line)]);
9088
}
@@ -160,7 +158,7 @@ void ExaMiniMD::init(int argc, char* argv[]) {
160158
} else {
161159
printf("\n");
162160
printf("Step Temp E_pair TotEng CPU\n");
163-
printf(" %i %lf %lf %lf %lf\n",step,T,PE,PE+KE,0.0);
161+
printf("%i %lf %lf %lf %lf\n",step,T,PE,PE+KE,0.0);
164162
}
165163
}
166164
}
@@ -221,11 +219,12 @@ void ExaMiniMD::run(int nsteps) {
221219
neighbor->create_neigh_list(system,binning,force->half_neigh,false);
222220
neigh_time += neigh_timer.seconds();
223221
} else {
224-
// Exchange Halo
222+
// Exchange Halo data
225223
comm_timer.reset();
226224
comm->update_halo();
227225
comm_time += comm_timer.seconds();
228226
}
227+
Kokkos::Experimental::DefaultRemoteMemorySpace::fence();
229228

230229
// Zero out forces
231230
force_timer.reset();
@@ -249,7 +248,7 @@ void ExaMiniMD::run(int nsteps) {
249248
integrator->final_integrate();
250249

251250
// On output steps print output
252-
if(step%input->thermo_rate==0) {
251+
//if(step%input->thermo_rate==0) {
253252
T_FLOAT T = temp.compute(system);
254253
T_FLOAT PE = pote.compute(system,binning,neighbor,force)/system->N;
255254
T_FLOAT KE = kine.compute(system)/system->N;
@@ -264,7 +263,7 @@ void ExaMiniMD::run(int nsteps) {
264263
last_time = time;
265264
}
266265
}
267-
}
266+
// }
268267

269268
if(input->dumpbinaryflag)
270269
dump_binary(step);

0 commit comments

Comments
 (0)