Skip to content
This repository was archived by the owner on Mar 20, 2023. It is now read-only.

Commit 554c250

Browse files
oluptonolupton
authored
Make legacy scopmath solvers header only, drop kinderiv.py and dimplic.cpp (#809)
* Solvers used for derivimplicit, sparse and euler methods were compiled as a separate library. These solvers were called from generated .mod to .cpp files. * But as GPU programming models don't support function pointers, we use a very convoluted way via macros (+ _kinderiv.py) to achieve similar functionality as function pointers. This also needed to use `acc routine seq` for all the solver routines. But this is not compatible when we want to support shared libraries for python wheels (see #141) * With this PR we make all solvers inline-able by making them part of headers * Finally drop kinderiv.py ! * Drop acc routine seq * Reallocate net receive buffers using ecalloc_align/free_memory for consistency. * fix-up test, update submodules * Make the NetReceiveBuffer initial size at least 8.
1 parent a964c7a commit 554c250

26 files changed

+1218
-1754
lines changed

‎coreneuron/CMakeLists.txt

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@ file(
3131
"utils/*.cpp"
3232
"utils/*/*.c"
3333
"utils/*/*.cpp")
34-
set(SCOPMATH_CODE_FILES
35-
"sim/scopmath/abort.cpp" "sim/scopmath/crout_thread.cpp" "sim/scopmath/newton_thread.cpp"
36-
"sim/scopmath/sparse_thread.cpp" "sim/scopmath/ssimplic_thread.cpp")
34+
set(SCOPMATH_CODE_FILES "sim/scopmath/abort.cpp" "sim/scopmath/crout_thread.cpp"
35+
"sim/scopmath/newton_thread.cpp")
3736
set(MPI_LIB_FILES "mpi/lib/mpispike.cpp" "mpi/lib/nrnmpi.cpp")
3837
set(MPI_CORE_FILES "mpi/core/nrnmpi_def_cinc.cpp")
3938
if(CORENRN_ENABLE_MPI)
@@ -44,19 +43,11 @@ file(COPY ${CORENEURON_PROJECT_SOURCE_DIR}/external/Random123/include/Random123
4443
DESTINATION ${CMAKE_BINARY_DIR}/include)
4544
list(APPEND CORENEURON_CODE_FILES ${PROJECT_BINARY_DIR}/coreneuron/config/config.cpp)
4645

47-
set(DIMPLIC_CODE_FILE "mechanism/mech/dimplic.cpp")
4846
set(ENGINEMECH_CODE_FILE "mechanism/mech/enginemech.cpp")
4947

5048
# for external mod files we need to generate modl_ref function in mod_func.c
5149
set(MODFUNC_PERL_SCRIPT "mechanism/mech/mod_func.c.pl")
5250

53-
# ~~~
54-
# as openacc limits passing function pointers as arguments, mod2c translated files
55-
# depend on a _kinderiv.h file that is constructed by kinderiv.py
56-
# ~~~
57-
set(KINDERIV_PYTHON_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kinderiv.py")
58-
set(KINDERIV_HEADER_FILE "${CMAKE_CURRENT_BINARY_DIR}/_kinderiv.h")
59-
6051
set(NMODL_UNITS_FILE "${CMAKE_BINARY_DIR}/share/mod2c/nrnunits.lib")
6152

6253
# =============================================================================
@@ -66,23 +57,12 @@ cpp_cc_build_time_copy(
6657
INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${MODFUNC_PERL_SCRIPT}"
6758
OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
6859
NO_TARGET)
69-
cpp_cc_build_time_copy(
70-
INPUT "${KINDERIV_PYTHON_SCRIPT}"
71-
OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/kinderiv.py"
72-
NO_TARGET)
73-
cpp_cc_build_time_copy(
74-
INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${DIMPLIC_CODE_FILE}"
75-
OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/dimplic.cpp"
76-
NO_TARGET)
7760
cpp_cc_build_time_copy(
7861
INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${ENGINEMECH_CODE_FILE}"
7962
OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp"
8063
NO_TARGET)
81-
set(nrnivmodl_core_dependencies
82-
"${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
83-
"${CMAKE_BINARY_DIR}/share/coreneuron/kinderiv.py"
84-
"${CMAKE_BINARY_DIR}/share/coreneuron/dimplic.cpp"
85-
"${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp")
64+
set(nrnivmodl_core_dependencies "${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
65+
"${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp")
8666
# Set up build rules that copy builtin mod files from
8767
# {source}/coreneuron/mechanism/mech/modfile/*.mod to {build_dir}/share/modfile/
8868
file(GLOB builtin_modfiles
@@ -150,17 +130,6 @@ set_property(
150130
APPEND
151131
PROPERTY COMPILE_DEFINITIONS "CORENRN_USE_LEGACY_UNITS=${CORENRN_USE_LEGACY_UNITS}")
152132

153-
# =============================================================================
154-
# run KINDERIV_PYTHON_SCRIPT to generate _kinderiv.h
155-
# =============================================================================
156-
add_custom_command(
157-
OUTPUT "${KINDERIV_HEADER_FILE}"
158-
DEPENDS ${NMODL_INBUILT_MOD_OUTPUTS} "${KINDERIV_PYTHON_SCRIPT}"
159-
COMMAND ${PYTHON_EXECUTABLE} ${KINDERIV_PYTHON_SCRIPT} ${CMAKE_CURRENT_BINARY_DIR}
160-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
161-
COMMENT "Generating kinderiv.h by inspecting MOD files")
162-
add_custom_target(kin_deriv_header DEPENDS "${KINDERIV_HEADER_FILE}")
163-
164133
# =============================================================================
165134
# create libraries
166135
# =============================================================================
@@ -275,9 +244,7 @@ set_target_properties(coreneuron PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
275244
# Suppress some compiler warnings. Note in GPU builds this library includes CUDA files.
276245
target_compile_options(coreneuron
277246
PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${CORENEURON_CXX_WARNING_SUPPRESSIONS}>)
278-
279-
# need to have _kinderiv.h for mod2c generated files and nrnivmodl-core and nmodl building
280-
add_dependencies(coreneuron kin_deriv_header nrnivmodl-core)
247+
add_dependencies(coreneuron nrnivmodl-core)
281248

282249
# scopmath is created separately for nrnivmodl-core workflow
283250
add_library(scopmath STATIC ${CORENEURON_HEADER_FILES} ${SCOPMATH_CODE_FILES})
@@ -400,8 +367,7 @@ install(
400367
PATTERN "*.h*"
401368
PATTERN "*.ipp"
402369
PATTERN "*.ispc")
403-
install(FILES ${KINDERIV_PYTHON_SCRIPT} ${MODFUNC_PERL_SCRIPT} ${DIMPLIC_CODE_FILE}
404-
${ENGINEMECH_CODE_FILE} DESTINATION share/coreneuron)
370+
install(FILES ${MODFUNC_PERL_SCRIPT} ${ENGINEMECH_CODE_FILE} DESTINATION share/coreneuron)
405371

406372
# copy mod2c/nmodl for nrnivmodl-core
407373
install(PROGRAMS ${CORENRN_MOD2CPP_BINARY} DESTINATION bin)

‎coreneuron/apps/corenrn_parameters.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
# =============================================================================
3-
# Copyright (c) 2016 - 2021 Blue Brain Project/EPFL
3+
# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL
44
#
55
# See top-level LICENSE file for details.
66
# =============================================================================.
@@ -184,13 +184,21 @@ void corenrn_parameters::parse(int argc, char** argv) {
184184
<< std::endl;
185185
app.exit(e);
186186
throw e;
187-
188187
} catch (const CLI::ParseError& e) {
189188
// use --help is also ParseError; in this case exit by showing all options
190189
app.exit(e);
191190
exit(0);
192191
}
193192

193+
#ifndef CORENEURON_ENABLE_GPU
194+
if (gpu) {
195+
std::cerr
196+
<< "Error: GPU support was not enabled at build time but GPU execution was requested."
197+
<< std::endl;
198+
exit(42);
199+
}
200+
#endif
201+
194202
// is user has asked for version info, print it and exit
195203
if (show_version) {
196204
std::cout << "CoreNEURON Version : " << cnrn_version() << std::endl;

‎coreneuron/gpu/nrn_acc_manager.cpp

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
# =============================================================================
3-
# Copyright (c) 2016 - 2021 Blue Brain Project/EPFL
3+
# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL
44
#
55
# See top-level LICENSE file for details.
66
# =============================================================================
@@ -626,44 +626,50 @@ void realloc_net_receive_buffer(NrnThread* nt, Memb_list* ml) {
626626
cnrn_target_delete(nrb->_nrb_index, nrb->_size);
627627
}
628628
#endif
629-
630-
// Reallocate host
629+
// Reallocate host buffers using ecalloc_align (as in phase2.cpp) and
630+
// free_memory (as in nrn_setup.cpp)
631+
auto const realloc = [old_size = nrb->_size, nrb](auto*& ptr, std::size_t extra_size = 0) {
632+
using T = std::remove_pointer_t<std::remove_reference_t<decltype(ptr)>>;
633+
static_assert(std::is_trivial<T>::value,
634+
"Only trivially constructible and copiable types are supported.");
635+
static_assert(std::is_same<decltype(ptr), T*&>::value,
636+
"ptr should be reference-to-pointer");
637+
auto* const new_data = static_cast<T*>(ecalloc_align((nrb->_size + extra_size), sizeof(T)));
638+
std::memcpy(new_data, ptr, (old_size + extra_size) * sizeof(T));
639+
free_memory(ptr);
640+
ptr = new_data;
641+
};
631642
nrb->_size *= 2;
632-
nrb->_pnt_index = (int*) erealloc(nrb->_pnt_index, nrb->_size * sizeof(int));
633-
nrb->_weight_index = (int*) erealloc(nrb->_weight_index, nrb->_size * sizeof(int));
634-
nrb->_nrb_t = (double*) erealloc(nrb->_nrb_t, nrb->_size * sizeof(double));
635-
nrb->_nrb_flag = (double*) erealloc(nrb->_nrb_flag, nrb->_size * sizeof(double));
636-
nrb->_displ = (int*) erealloc(nrb->_displ, (nrb->_size + 1) * sizeof(int));
637-
nrb->_nrb_index = (int*) erealloc(nrb->_nrb_index, nrb->_size * sizeof(int));
638-
643+
realloc(nrb->_pnt_index);
644+
realloc(nrb->_weight_index);
645+
realloc(nrb->_nrb_t);
646+
realloc(nrb->_nrb_flag);
647+
realloc(nrb->_displ, 1);
648+
realloc(nrb->_nrb_index);
639649
#ifdef CORENEURON_ENABLE_GPU
640650
if (nt->compute_gpu) {
641-
int *d_weight_index, *d_pnt_index, *d_displ, *d_nrb_index;
642-
double *d_nrb_t, *d_nrb_flag;
643-
644651
// update device copy
645652
nrn_pragma_acc(update device(nrb));
646653
nrn_pragma_omp(target update to(nrb));
647654

648-
NetReceiveBuffer_t* d_nrb = cnrn_target_deviceptr(nrb);
649-
655+
NetReceiveBuffer_t* const d_nrb{cnrn_target_deviceptr(nrb)};
650656
// recopy the vectors in the buffer
651-
d_pnt_index = cnrn_target_copyin(nrb->_pnt_index, nrb->_size);
657+
int* const d_pnt_index{cnrn_target_copyin(nrb->_pnt_index, nrb->_size)};
652658
cnrn_target_memcpy_to_device(&(d_nrb->_pnt_index), &d_pnt_index);
653659

654-
d_weight_index = cnrn_target_copyin(nrb->_weight_index, nrb->_size);
660+
int* const d_weight_index{cnrn_target_copyin(nrb->_weight_index, nrb->_size)};
655661
cnrn_target_memcpy_to_device(&(d_nrb->_weight_index), &d_weight_index);
656662

657-
d_nrb_t = cnrn_target_copyin(nrb->_nrb_t, nrb->_size);
663+
double* const d_nrb_t{cnrn_target_copyin(nrb->_nrb_t, nrb->_size)};
658664
cnrn_target_memcpy_to_device(&(d_nrb->_nrb_t), &d_nrb_t);
659665

660-
d_nrb_flag = cnrn_target_copyin(nrb->_nrb_flag, nrb->_size);
666+
double* const d_nrb_flag{cnrn_target_copyin(nrb->_nrb_flag, nrb->_size)};
661667
cnrn_target_memcpy_to_device(&(d_nrb->_nrb_flag), &d_nrb_flag);
662668

663-
d_displ = cnrn_target_copyin(nrb->_displ, nrb->_size + 1);
669+
int* const d_displ{cnrn_target_copyin(nrb->_displ, nrb->_size + 1)};
664670
cnrn_target_memcpy_to_device(&(d_nrb->_displ), &d_displ);
665671

666-
d_nrb_index = cnrn_target_copyin(nrb->_nrb_index, nrb->_size);
672+
int* const d_nrb_index{cnrn_target_copyin(nrb->_nrb_index, nrb->_size)};
667673
cnrn_target_memcpy_to_device(&(d_nrb->_nrb_index), &d_nrb_index);
668674
}
669675
#endif
@@ -947,8 +953,11 @@ void delete_nrnthreads_on_device(NrnThread* threads, int nthreads) {
947953
#ifdef CORENEURON_ENABLE_GPU
948954
for (int i = 0; i < nthreads; i++) {
949955
NrnThread* nt = threads + i;
950-
cnrn_target_delete(nt->_fornetcon_weight_perm);
951-
cnrn_target_delete(nt->_fornetcon_perm_indices);
956+
if (!nt->compute_gpu) {
957+
continue;
958+
}
959+
cnrn_target_delete(nt->_fornetcon_weight_perm, nt->_fornetcon_weight_perm_size);
960+
cnrn_target_delete(nt->_fornetcon_perm_indices, nt->_fornetcon_perm_indices_size);
952961
{
953962
TrajectoryRequests* tr = nt->trajec_requests;
954963
if (tr) {

‎coreneuron/io/phase2.cpp

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -482,12 +482,8 @@ void Phase2::set_net_send_buffer(Memb_list** ml_list, const std::vector<int>& pn
482482
ml->_net_receive_buffer = nrb;
483483
nrb->_pnt_offset = pnt_offset[type];
484484

485-
// begin with a size of 5% of the number of instances
486-
nrb->_size = ml->nodecount;
487-
// or at least 8
488-
nrb->_size = std::max(8, nrb->_size);
489-
// but not more than nodecount
490-
nrb->_size = std::min(ml->nodecount, nrb->_size);
485+
// begin with a size equal to the number of instances, or at least 8
486+
nrb->_size = std::max(8, ml->nodecount);
491487
nrb->_pnt_index = (int*) ecalloc_align(nrb->_size, sizeof(int));
492488
nrb->_displ = (int*) ecalloc_align(nrb->_size + 1, sizeof(int));
493489
nrb->_nrb_index = (int*) ecalloc_align(nrb->_size, sizeof(int));

‎coreneuron/kinderiv.py

Lines changed: 0 additions & 141 deletions
This file was deleted.

‎coreneuron/mechanism/mech/cfile/scoplib.h

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)