ESyS-Particle
4.0.1
|
00001 00002 // // 00003 // Copyright (c) 2003-2011 by The University of Queensland // 00004 // Earth Systems Science Computational Centre (ESSCC) // 00005 // http://www.uq.edu.au/esscc // 00006 // // 00007 // Primary Business: Brisbane, Queensland, Australia // 00008 // Licensed under the Open Software License version 3.0 // 00009 // http://www.opensource.org/licenses/osl-3.0.php // 00010 // // 00012 00013 #include "Parallel/GetRef_cmd.h" 00014 #include <map> 00015 #include <set> 00016 00017 template <typename TmplVisitor> 00018 void CLatticeMaster::visitMeshFaceReferences(const string &meshName) 00019 { 00020 throw std::runtime_error("CLatticeMaster::visitMeshFaceReferences: Not implemented."); 00021 } 00022 00023 template <typename TmplVisitor> 00024 void CLatticeMaster::visitMesh2dNodeReferences(const string &meshName, TmplVisitor &visitor) 00025 { 00026 console.XDebug()<<"CLatticeMaster::visitMesh2dNodeReferences( " << meshName << ")\n"; 00027 GetNodeRefCommand cmd(getGlobalRankAndComm(),meshName); 00028 // broadcast command 00029 cmd.broadcast(); 00030 00031 // receive data (multimap) 00032 std::multimap<int,int> ref_mmap; 00033 m_tml_global_comm.gather(ref_mmap); 00034 // collate into set 00035 std::set<int> ref_set; //== this is the set of node ids == 00036 for(std::multimap<int,int>::iterator iter=ref_mmap.begin(); 00037 iter!=ref_mmap.end(); 00038 iter++){ 00039 ref_set.insert(iter->second); 00040 } 00041 for (std::set<int>::const_iterator it = ref_set.begin(); it != ref_set.end(); it++) 00042 { 00043 visitor.visitNodeRef(*it); 00044 } 00045 console.XDebug()<<"end CLatticeMaster::visitMesh2dNodeReferences()\n"; 00046 } 00047 00048 template <typename TmplVisitor> 00049 void CLatticeMaster::visitMesh2dEdgeStress(const string &meshName, TmplVisitor &visitor) 00050 { 00051 console.XDebug()<<"CLatticeMaster::visitMesh2dEdgeStress( " << meshName << ")\n"; 00052 std::multimap<int,pair<int,Vec3> > temp_mm; 00053 std::map<int,Vec3> data; //=== map of id, value === 00054 00055 BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETMESH2DSTRESS); 00056 cmd.append(meshName.c_str()); 00057 cmd.broadcastCommand(); 00058 cmd.broadcastBuffer(); 00059 00060 // get data from slaves 00061 m_tml_global_comm.gather(temp_mm); 00062 00063 // add data together 00064 for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin(); 00065 iter!=temp_mm.end(); 00066 iter++){ 00067 if(data.find((iter->second).first)==data.end()){ // id not in data -> insert 00068 data.insert(iter->second); 00069 } else { // id is in data -> add 00070 data[(iter->second).first]+=(iter->second).second; 00071 } 00072 } 00073 00074 for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++) 00075 { 00076 visitor.visitRefStressPair(it->first, it->second); 00077 } 00078 00079 cmd.wait("visitMesh2dEdgeStress"); 00080 console.XDebug()<<"end CLatticeMaster::visitMesh2dEdgeStress()\n"; 00081 } 00082 00083 template <typename TmplVisitor> 00084 void CLatticeMaster::visitTriMeshFaceForce( 00085 const string &meshName, 00086 TmplVisitor &visitor 00087 ) 00088 { 00089 console.XDebug()<<"CLatticeMaster::visitTriMeshFaceForce( " << meshName << ")\n"; 00090 std::multimap<int,pair<int,Vec3> > temp_mm; 00091 std::map<int,Vec3> data; //=== map of id, value === 00092 00093 BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETTRIMESHFORCE); 00094 cmd.append(meshName.c_str()); 00095 cmd.broadcastCommand(); 00096 cmd.broadcastBuffer(); 00097 00098 // get data from slaves 00099 m_tml_global_comm.gather(temp_mm); 00100 00101 // add data together 00102 for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin(); 00103 iter!=temp_mm.end(); 00104 iter++){ 00105 if(data.find((iter->second).first)==data.end()){ // id not in data -> insert 00106 data.insert(iter->second); 00107 } else { // id is in data -> add 00108 data[(iter->second).first]+=(iter->second).second; 00109 } 00110 } 00111 00112 for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++) 00113 { 00114 visitor.visitRefForcePair(it->first, it->second); 00115 } 00116 00117 cmd.wait("visitTriMeshFaceStress"); 00118 console.XDebug()<<"end CLatticeMaster::visitTriMeshFaceForce()\n"; 00119 } 00120 00121 template <typename TmplVisitor, typename TmplParticle> 00122 void CLatticeMaster::visitParticlesOfType( 00123 const IdVector &particleIdVector, 00124 TmplVisitor &visitor 00125 ) 00126 { 00127 console.Debug() << "CLatticeMaster::visitParticlesOfType: enter\n"; 00128 typedef std::multimap<int,TmplParticle> ParticleMMap; 00129 ParticleMMap particleMMap; 00130 00131 console.Debug() 00132 << "CLatticeMaster::visitParticlesOfType: broadcasting command\n"; 00133 BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETIDPARTICLEDATA); 00134 cmd.broadcastCommand(); 00135 00136 console.Debug() 00137 << "CLatticeMaster::visitParticlesOfType: broadcasting particle id's\n"; 00138 m_tml_global_comm.broadcast_cont(particleIdVector); 00139 00140 console.Debug() 00141 << "CLatticeMaster::visitParticlesOfType:" 00142 << " gathering particle data from workers\n"; 00143 m_tml_global_comm.gather_packed(particleMMap); 00144 console.Debug() 00145 << "CLatticeMaster::visitParticlesOfType:" 00146 << " gathered " << particleMMap.size() << " particles\n"; 00147 00148 console.Debug() 00149 << "CLatticeMaster::visitParticlesOfType:" 00150 << " visiting particle data\n"; 00151 for( 00152 typename ParticleMMap::iterator iter=particleMMap.begin(); 00153 iter != particleMMap.end(); 00154 iter++ 00155 ) 00156 { 00157 iter->second.visit(visitor); 00158 } 00159 00160 cmd.wait("visitParticles"); 00161 console.Debug() << "CLatticeMaster::visitParticlesOfType: exit\n"; 00162 } 00163 00164 template <typename TmplVisitor> 00165 void CLatticeMaster::visitParticles( 00166 const IdVector &particleIdVector, 00167 TmplVisitor &visitor 00168 ) 00169 { 00170 console.Debug() << "CLatticeMaster::visitParticles: enter\n"; 00171 00172 if (m_particle_type == "Basic") 00173 { 00174 visitParticlesOfType<TmplVisitor,CParticle>(particleIdVector, visitor); 00175 } 00176 else if (m_particle_type == "Rot") 00177 { 00178 visitParticlesOfType<TmplVisitor,CRotParticle>(particleIdVector, visitor); 00179 } 00180 else if (m_particle_type == "RotVi") 00181 { 00182 visitParticlesOfType<TmplVisitor,CRotParticleVi>(particleIdVector, visitor); 00183 } 00184 else if (m_particle_type == "RotTherm") 00185 { 00186 visitParticlesOfType<TmplVisitor,CRotThermParticle>(particleIdVector, visitor); 00187 } 00188 else 00189 { 00190 throw 00191 std::runtime_error( 00192 std::string("Unknown particle type: ") + m_particle_type 00193 ); 00194 } 00195 console.Debug() << "CLatticeMaster::visitParticles: exit\n"; 00196 } 00197 00198 template <class TmplIterator, class TmplParticle> 00199 void CLatticeMaster::addParticles(TmplIterator &particleIt) 00200 { 00201 CMPIBarrier barrier(m_global_comm); 00202 CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank); 00203 00204 // cast storage pointer to correct type 00205 vector<TmplParticle> particleVector; 00206 console.XDebug() 00207 << "CLatticeMaster::addParticles:" 00208 << " Reserving vector memory for particles.\n"; 00209 00210 const int numBroadcastParticles = 50000; 00211 particleVector.reserve(numBroadcastParticles); 00212 console.XDebug() 00213 << "CLatticeMaster::addParticles:" 00214 << " Beginning add-particle loop..." << "\n"; 00215 for (int i = 0; particleIt.hasNext(); i++) { 00216 const TmplParticle particle(particleIt.next()); 00217 00218 particleVector.push_back(particle); 00219 if (((i+1) % 5000) == 0) { 00220 console.XDebug() 00221 << "CLatticeMaster::addParticles:" 00222 << "Adding particle with id " 00223 << particleVector.rbegin()->getID() << "\n"; 00224 } 00225 00226 if (((i+1) % numBroadcastParticles) == 0) 00227 { 00228 console.XDebug() << "CLatticeMaster::addParticles:" 00229 << " Broadcasting receive cmd...." << "\n"; 00230 cmd_buffer.broadcast(CMD_RECEIVEPARTICLES); 00231 console.XDebug() 00232 << "CLatticeMaster::addParticles: Broadcasting particles...." << "\n"; 00233 m_tml_global_comm.broadcast_cont_packed(particleVector); 00234 barrier.wait("CLatticeMaster::addParticles: Post particle broadcast."); 00235 barrier.wait("CLatticeMaster::addParticles: Post Command."); 00236 particleVector.clear(); 00237 particleVector.reserve(numBroadcastParticles); 00238 } 00239 } 00240 console.XDebug() 00241 << "CLatticeMaster::addParticles: Done add-particle loop..." << "\n"; 00242 console.XDebug() 00243 << "CLatticeMaster::addParticles: Broadcasting final particle-receive cmd" 00244 << "\n"; 00245 cmd_buffer.broadcast(CMD_RECEIVEPARTICLES); 00246 console.XDebug() << "CLatticeMaster::addParticles:" 00247 << " Broadcasting final set of particles...\n"; 00248 m_tml_global_comm.broadcast_cont_packed(particleVector); 00249 barrier.wait("Post final particle broadcast."); 00250 barrier.wait("Post final particle-broadcast command."); 00251 // -- build ntable 00252 console.XDebug() 00253 << "CLatticeMaster::addParticles: " 00254 << "Building ntable (searchNeighbours)...\n"; 00255 searchNeighbors(true); 00256 console.XDebug() 00257 << "CLatticeMaster::addParticles: exit\n"; 00258 00259 } 00260 00261 template <class TmplIterator> 00262 void CLatticeMaster::addConnections(TmplIterator &connectionIt) 00263 { 00264 console.XDebug() 00265 << "CLatticeMaster::addConnections: enter\n"; 00266 00267 CMPIBarrier barrier(m_global_comm); 00268 CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank); 00269 00270 const int numBroadcastConnections = 100000; 00271 vector<int> connectionBuffer; 00272 connectionBuffer.reserve(numBroadcastConnections); 00273 int i = 0; 00274 for (; connectionIt.hasNext(); i++) 00275 { 00276 typename TmplIterator::value_type data = connectionIt.next(); 00277 connectionBuffer.push_back(data.getTag()); 00278 connectionBuffer.push_back(data.getP1Id()); 00279 connectionBuffer.push_back(data.getP2Id()); 00280 if ((i+1) % 50000 == 0) 00281 { 00282 console.XDebug() << "Adding connection number " << i << "\n"; 00283 } 00284 if ((i+1) % numBroadcastConnections == 0) 00285 { 00286 console.XDebug() << "CLatticeMaster::addConnections:" 00287 << " Broadcasting receive cmd...." << "\n"; 00288 cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS); 00289 console.XDebug() 00290 << "CLatticeMaster::addConnections: Broadcasting connections...." << "\n"; 00291 m_tml_global_comm.broadcast_cont_packed(connectionBuffer); 00292 barrier.wait("CLatticeMaster::addConnections: Post connection broadcast."); 00293 barrier.wait("CLatticeMaster::addConnections: Post Command."); 00294 connectionBuffer.clear(); 00295 connectionBuffer.reserve(numBroadcastConnections); 00296 } 00297 //m_temp_conn[data.getTag()].push_back(data.getP1Id()); 00298 //m_temp_conn[data.getTag()].push_back(data.getP2Id()); 00299 } 00300 console.XDebug() 00301 << "CLatticeMaster::addConnections: Done add-connection loop..." << "\n"; 00302 console.XDebug() 00303 << "CLatticeMaster::addConnections: Broadcasting final connection-receive cmd" 00304 << "\n"; 00305 cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS); 00306 console.XDebug() << "CLatticeMaster::addConnections:" 00307 << " Broadcasting final set of connections...\n"; 00308 m_tml_global_comm.broadcast_cont_packed(connectionBuffer); 00309 barrier.wait("Post final connection broadcast."); 00310 barrier.wait("Post final connection-broadcast command."); 00311 00312 console.XDebug()<< "Added " << i << " connections..." << "\n"; 00313 // -- build ntable 00314 searchNeighbors(true); 00315 console.XDebug() 00316 << "CLatticeMaster::addConnections: exit\n"; 00317 }