diff --git a/coreneuron/io/nrn2core_direct.h b/coreneuron/io/nrn2core_direct.h index 3c4990a05..f7f190c47 100644 --- a/coreneuron/io/nrn2core_direct.h +++ b/coreneuron/io/nrn2core_direct.h @@ -30,7 +30,8 @@ extern int (*nrn2core_get_dat1_)(int tid, int& n_presyn, int& n_netcon, int*& output_gid, - int*& netcon_srcgid); + int*& netcon_srcgid, + std::vector& netcon_negsrcgid_tid); extern int (*nrn2core_get_dat2_1_)(int tid, int& ngid, diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index 28fa1de07..c3d2e9df5 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -113,6 +113,14 @@ void (*nrn2core_all_weights_return_)(std::vector& weights); // encode the thread number into the negative gid // (i.e -ith - nth*(type +1000*index)) failed due to not large enough // integer domain size. +// Note that for file transfer it is an error if a negative srcgid is +// not in the same thread as the target. This is because there it may +// not be the case that threads in a NEURON process end up on same process +// in CoreNEURON. NEURON will raise an error if this +// is the case. However, for direct memory transfer, it is allowed that +// a negative srcgid may be in a different thread than the target. So +// nrn2core_get_dat1 has a last arg netcon_negsrcgid_tid that specifies +// for the negative gids in netcon_srcgid (in that order) the source thread. // // _2.dat // n_output n_real_output, nnode @@ -178,7 +186,11 @@ std::map gid2in; std::vector netcon_in_presyn_order_; /// Only for setup vector of netcon source gids -std::vector netcon_srcgid; +std::vector nrnthreads_netcon_srcgid; + +/// If a nrnthreads_netcon_srcgid is negative, need to determine the thread when +/// in order to use the correct neg_gid2out[tid] map +std::vector > nrnthreads_netcon_negsrcgid_tid; /* read files.dat file and distribute cellgroups to all mpi ranks */ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { @@ -275,8 +287,11 @@ void determine_inputpresyn() { NrnThread& nt = nrn_threads[ith]; // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count nt.n_input_presyn = 0; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; if (gid >= 0) { /// If PreSyn or InputPreSyn is already in the map auto gid2out_it = gid2out.find(gid); @@ -299,8 +314,12 @@ void determine_inputpresyn() { inputpresyn_.push_back(psi); ++nt.n_input_presyn; } else { - auto gid2out_it = neg_gid2out[nt.id].find(gid); - if (gid2out_it != neg_gid2out[nt.id].end()) { + int tid = nt.id; + if (!negsrcgid_tid.empty()) { + tid = negsrcgid_tid[i_tid++]; + } + auto gid2out_it = neg_gid2out[tid].find(gid); + if (gid2out_it != neg_gid2out[tid].end()) { /// Increase negative PreSyn count ++gid2out_it->second->nc_cnt_; } @@ -361,16 +380,23 @@ void determine_inputpresyn() { // fill the netcon_in_presyn_order and recompute nc_cnt_ // note that not all netcon_in_presyn will be filled if there are netcon - // with no presyn (ie. netcon_srcgid[nt.id][i] = -1) but that is ok since they are + // with no presyn (ie. nrnthreads_netcon_srcgid[nt.id][i] = -1) but that is ok since they are // only used via ps.nc_index_ and ps.nc_cnt_; for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; + int tid = ith; + if (!negsrcgid_tid.empty() && gid < -1) { + tid = negsrcgid_tid[i_tid++]; + } PreSyn* ps; InputPreSyn* psi; - netpar_tid_gid2ps(ith, gid, &ps, &psi); + netpar_tid_gid2ps(tid, gid, &ps, &psi); if (ps) { netcon_in_presyn_order_[ps->nc_index_ + ps->nc_cnt_] = nc; ++ps->nc_cnt_; @@ -390,10 +416,11 @@ void determine_inputpresyn() { /// Clean up void nrn_setup_cleanup() { for (int ith = 0; ith < nrn_nthread; ++ith) { - if (netcon_srcgid[ith]) - delete[] netcon_srcgid[ith]; + if (nrnthreads_netcon_srcgid[ith]) + delete[] nrnthreads_netcon_srcgid[ith]; } - netcon_srcgid.clear(); + nrnthreads_netcon_srcgid.clear(); + nrnthreads_netcon_negsrcgid_tid.clear(); neg_gid2out.clear(); } @@ -449,9 +476,9 @@ void nrn_setup(const char* filesdat, /// std::map gid2out; gid2out.clear(); - netcon_srcgid.resize(nrn_nthread); + nrnthreads_netcon_srcgid.resize(nrn_nthread); for (int i = 0; i < nrn_nthread; ++i) - netcon_srcgid[i] = nullptr; + nrnthreads_netcon_srcgid[i] = nullptr; // gap junctions if (nrn_have_gaps) { @@ -470,6 +497,7 @@ void nrn_setup(const char* filesdat, nrn_partrans::gap_mpi_setup(userParams.ngroup); } + nrnthreads_netcon_negsrcgid_tid.resize(nrn_nthread); if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { @@ -481,7 +509,7 @@ void nrn_setup(const char* filesdat, }); } - // from the gid2out map and the netcon_srcgid array, + // from the gid2out map and the nrnthreads_netcon_srcgid array, // fill the gid2in, and from the number of entries, // allocate the process wide InputPreSyn array determine_inputpresyn(); diff --git a/coreneuron/io/phase1.cpp b/coreneuron/io/phase1.cpp index dd96bc4e1..5055ee98e 100644 --- a/coreneuron/io/phase1.cpp +++ b/coreneuron/io/phase1.cpp @@ -10,7 +10,8 @@ int (*nrn2core_get_dat1_)(int tid, int& n_presyn, int& n_netcon, int*& output_gid, - int*& netcon_srcgid); + int*& netcon_srcgid, + std::vector& netcon_negsrcgid_tid); namespace coreneuron { void Phase1::read_file(FileHandler& F) { @@ -20,6 +21,8 @@ void Phase1::read_file(FileHandler& F) { this->output_gids = F.read_vector(n_presyn); this->netcon_srcgids = F.read_vector(n_netcon); + // For file mode transfer, it is not allowed that negative gids exist + // in different threads. So this->netcon_tids remains clear. F.close(); } @@ -32,7 +35,7 @@ void Phase1::read_direct(int thread_id) { // TODO : check error codes for NEURON - CoreNEURON communication int valid = - (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid); + (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid, this->netcon_negsrcgid_tid); if (!valid) { return; } @@ -47,9 +50,12 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { nt.n_presyn = this->output_gids.size(); nt.n_netcon = this->netcon_srcgids.size(); - netcon_srcgid[nt.id] = new int[nt.n_netcon]; + nrnthreads_netcon_srcgid[nt.id] = new int[nt.n_netcon]; std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(), - netcon_srcgid[nt.id]); + nrnthreads_netcon_srcgid[nt.id]); + + // netcon_negsrcgid_tid is empty if file transfer or single thread + coreneuron::nrnthreads_netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; nt.netcons = new NetCon[nt.n_netcon]; nt.presyns_helper = (PreSynHelper*)ecalloc_align(nt.n_presyn, sizeof(PreSynHelper)); diff --git a/coreneuron/io/phase1.hpp b/coreneuron/io/phase1.hpp index 7d5ec5a05..8ab8d999b 100644 --- a/coreneuron/io/phase1.hpp +++ b/coreneuron/io/phase1.hpp @@ -18,6 +18,7 @@ class Phase1 { private: std::vector output_gids; std::vector netcon_srcgids; + std::vector netcon_negsrcgid_tid; // entries only for negative srcgids }; } // namespace coreneuron diff --git a/coreneuron/network/netpar.cpp b/coreneuron/network/netpar.cpp index 7c815690e..13fdf9e6d 100644 --- a/coreneuron/network/netpar.cpp +++ b/coreneuron/network/netpar.cpp @@ -641,10 +641,17 @@ double set_mindelay(double maxdelay) { for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; bool chk = false; // ignore nc.delay_ - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; + int tid = ith; + if (!negsrcgid_tid.empty() && gid < -1) { + tid = negsrcgid_tid[i_tid++]; + } PreSyn* ps; InputPreSyn* psi; netpar_tid_gid2ps(ith, gid, &ps, &psi); diff --git a/coreneuron/nrniv/nrniv_decl.h b/coreneuron/nrniv/nrniv_decl.h index 2d053962f..6f92c2970 100644 --- a/coreneuron/nrniv/nrniv_decl.h +++ b/coreneuron/nrniv/nrniv_decl.h @@ -48,8 +48,11 @@ extern std::map gid2in; /// InputPreSyn.nc_index_ to + InputPreSyn.nc_cnt_ give the NetCon* extern std::vector netcon_in_presyn_order_; -/// Only for setup vector of netcon source gids -extern std::vector netcon_srcgid; +/// Only for setup vector of netcon source gids and mindelay determination +extern std::vector nrnthreads_netcon_srcgid; +/// Companion to nrnthreads_netcon_srcgid when src gid is negative to allow +/// determination of the NrnThread of the source PreSyn. +extern std::vector > nrnthreads_netcon_negsrcgid_tid; extern void mk_mech(const char* path); extern void set_globals(const char* path, bool cli_global_seed, int cli_global_seed_value);