ESyS-Particle  4.0.1
LatticeMaster.h
00001 
00002 //                                                         //
00003 // Copyright (c) 2003-2011 by The University of Queensland //
00004 // Earth Systems Science Computational Centre (ESSCC)      //
00005 // http://www.uq.edu.au/esscc                              //
00006 //                                                         //
00007 // Primary Business: Brisbane, Queensland, Australia       //
00008 // Licensed under the Open Software License version 3.0    //
00009 // http://www.opensource.org/licenses/osl-3.0.php          //
00010 //                                                         //
00012 
00013 #ifndef __LATTICEMASTER_H
00014 #define __LATTICEMASTER_H
00015 
00016 //--- Project includes ---
00017 
00018 #include "Parallel/mpibuf.h"
00019 #include "Parallel/mpivbuf.h"
00020 #include "Parallel/LatticeParam.h"
00021 #include "Parallel/RankAndComm.h"
00022 #include "Parallel/sublattice_cmd.h"
00023 
00024 #include "config.h"
00025 
00026 #include "Foundation/console.h"
00027 #include "Foundation/Runnable.h"
00028 
00029 #include "Fields/FieldMaster.h"
00030 #include "Fields/MaxTrigger.h"
00031 
00032 #include "Model/Damping.h"
00033 #include "Model/LocalDamping.h"
00034 #include "Model/ABCDampingIGP.h"
00035 #include "Model/Particle.h"
00036 #include "Model/RotParticle.h"
00037 #include "Model/RotParticleVi.h"
00038 #include "Model/RotThermParticle.h"
00039 #include "Model/ElasticInteraction.h"
00040 #include "Model/FrictionInteraction.h"
00041 #include "Model/FractalFriction.h"
00042 #include "Model/AdhesiveFriction.h"
00043 #include "Model/HertzianElasticInteraction.h"
00044 #include "Model/HertzianViscoElasticFrictionInteraction.h"
00045 #include "Model/HertzianViscoElasticInteraction.h"
00046 #include "Model/LinearDashpotInteraction.h"
00047 #include "Model/MeshData.h"
00048 #include "Model/ETriMeshIP.h"
00049 #include "Model/BTriMeshIP.h"
00050 #include "Model/BMesh2DIP.h"
00051 #include "Model/BondedInteraction.h"
00052 #include "Model/CappedBondedInteraction.h"
00053 #include "Model/RotFricInteraction.h"
00054 #include "Model/RotBondedInteraction.h"
00055 #include "Model/RotElasticInteraction.h"
00056 #include "Model/RotThermFricInteraction.h"
00057 #include "Model/RotThermBondedInteraction.h"
00058 #include "Model/RotThermElasticInteraction.h"
00059 #include "Model/BodyForceGroup.h"
00060 #include "Model/EWallInteractionGroup.h"
00061 #include "Model/BWallInteractionGroup.h"
00062 #include "Model/ViscWallIG.h"
00063 #include "Model/SoftBWallInteractionGroup.h"
00064 
00065 #include <boost/filesystem/path.hpp>
00066 
00067 //--- MPI includes ---
00068 #include <mpi.h>
00069 
00070 //--- TML includes ---
00071 #include "tml/comm/comm_world.h"
00072 
00073 // -- STL includes --
00074 #include <vector>
00075 #include <list>
00076 #include <map>
00077 #include <utility>
00078 #include <string>
00079 
00080 // forward decls. 
00081 // includes are in the .cpp
00082 class CheckPointController;
00083 
00084 namespace esys
00085 {
00086   namespace lsm
00087   {
00088     class GeometryInfo;
00089     class BodyForceIGP;
00090   }
00091 }
00092 
00100 class MpiWTimers;
00101 
00102 namespace esys
00103 {
00104   namespace lsm
00105   {
00106     typedef std::vector<int> IntVector;
00107   }
00108 }
00109 
00110 class SpawnCmd
00111 {
00112 public:
00113   typedef std::vector<std::string> StringVector;
00114   
00115   SpawnCmd();
00116 
00117   const std::string &getExe() const;
00118 
00119   boost::filesystem::path getExePath() const;
00120 
00121   void setExe(const std::string &exeName);
00122   
00123   const StringVector &getArgVector() const;
00124 
00125   void setArgVector(const StringVector &argVector);
00126 
00127   std::string getCmdLine() const;
00128 
00129 private:
00130   std::string  m_exeName;
00131   StringVector m_argVector;
00132 };
00133 
00134 class CLatticeMaster 
00135 {
00136  public:
00137     typedef std::vector<esys::lsm::Runnable *> RunnableVector;
00138     typedef std::pair<int, int>                ParticleIdPair;
00139     typedef std::vector<ParticleIdPair>        ParticleIdPairVector;
00140     typedef std::vector<MeshNodeData> MeshNodeDataVector;
00141     typedef std::vector<MeshTriData> MeshTriDataVector;
00142     typedef std::pair<MeshNodeDataVector,MeshTriDataVector> TriMeshDataPair;
00143  
00144  private:
00145     static std::string s_workerExeName; // name of the worker executable
00146 
00147     std::string                           m_timingFileName;
00148     MpiWTimers                            *m_pTimers;
00149     CheckPointController                  *m_pCheckPointController; // for restart checkpoints
00150     CheckPointController                  *m_pSnapShotController; // for viz/analysis dumps
00151     esys::lsm::CLatticeParam::ProcessDims m_processDims;
00152 
00153  protected:
00154     typedef std::vector<int> ConnIdVector;
00155     map<int,ConnIdVector> m_temp_conn;
00156     vector<AFieldMaster*> m_save_fields;
00157 
00158     double m_x_max,m_x_min,m_y_max,m_y_min,m_z_max,m_z_min;
00159     int m_global_rank;
00160     int m_global_size;
00161     int m_max_ts;
00162     int m_center_id;
00163     double m_total_time;
00164     int m_t ;
00165     double m_dt;
00166     bool m_isInitialized ;
00167     bool m_first_time;
00168     std::string m_particle_type;
00169 
00170     RunnableVector m_preRunnableVector;
00171     RunnableVector m_postRunnableVector;
00172 
00173     TML_Comm m_tml_global_comm;
00174     MPI_Comm m_global_comm;
00175 
00176     void runRunnables(RunnableVector::iterator begin, RunnableVector::iterator end);
00177     void runPreRunnables();
00178     void runPostRunnables();
00179 
00180     void makeWorkerComm();
00181 
00182     void saveTimingData();
00183     TriMeshDataPair readTriMesh(const std::string &fileName,int);
00184     TriMeshDataPair readTriMesh(const std::string &fileName);
00185     void readAndDistributeMesh2D(const std::string&,int);
00186 
00187   MpiRankAndComm getGlobalRankAndComm() const
00188   {
00189     return MpiRankAndComm(m_global_rank, m_global_comm);
00190   }
00191 
00192   void disconnectWorkerProcesses();
00193 
00194 public:
00195     CLatticeMaster();
00196     ~CLatticeMaster();
00197 
00198     std::string getLsmVersion() const
00199     {
00200       return std::string(PACKAGE_VERSION);
00201     }
00202 
00203     int getNumWorkerProcesses() const;
00204 
00205     int getTimeStep() const {return m_t;}
00206     double getTimeStepSize() const {return m_dt;}
00207     void setTimeStepSize(double dt);
00208 
00209     void init();
00210 
00219     /* void setWorkerSpawnCmd( */
00220 /*       const std::string &exe, */
00221 /*       const SpawnCmd::StringVector &argVector */
00222 /*     ); */
00223 
00228     //const SpawnCmd &getWorkerSpawnCmd();
00229 
00233     void spawnSlaves(int numWorkers);
00234     void run();
00235     void runInit();
00236     void runOneStep();
00237     void runEnd();
00238     void oneStep();
00239     void searchNeighbors(bool);
00240     bool checkNeighbors();
00241     void updateInteractions();
00242     void addBondedIG(const CBondedIGP&);
00243     void addCappedBondedIG(int,const std::string&,double,double,double);
00244     void addShortBondedIG(int,const std::string&,double,double);
00245     
00246     void addPairIG(const CElasticIGP &prms);
00247     void addPairIG(const CFrictionIGP &prms);
00248     void addPairIG(const FractalFrictionIGP &prms);
00249     void addPairIG(const CAdhesiveFrictionIGP &prms);
00250     void addPairIG(const CRotElasticIGP &prms);
00251     void addPairIG(const CRotFrictionIGP &prms);
00252     void addPairIG(const CHertzianElasticIGP &prms);
00253     void addPairIG(const CHertzianViscoElasticFrictionIGP &prms);
00254     void addPairIG(const CHertzianViscoElasticIGP &prms);
00255     void addPairIG(const CLinearDashpotIGP &prms);
00256     void addPairIG(const CRotThermElasticIGP &prms);
00257     void addPairIG(const CRotThermFrictionIGP &prms);
00258     void addTaggedPairIG(const CRotFrictionIGP &prms,int,int,int,int);
00259     void addTaggedPairIG(const CFrictionIGP &prms,int,int,int,int);
00260     void addTaggedPairIG(const CHertzianElasticIGP &prms,int,int,int,int);
00261     void addTaggedPairIG(const CHertzianViscoElasticFrictionIGP &prms,int,int,int,int);
00262     void addTaggedPairIG(const CHertzianViscoElasticIGP &prms,int,int,int,int);
00263     void addTaggedPairIG(const CLinearDashpotIGP &prms,int,int,int,int);
00264     void addTaggedPairIG(const CRotElasticIGP &prms,int,int,int,int);
00265     void addTaggedPairIG(const CElasticIGP &prms,int,int,int,int);
00266 
00267     void removeIG(const std::string&);
00268     
00269     void readAndDistributeTriMesh(const std::string&,const std::string&,int);
00270     void readAndDistributeTriMesh(const std::string&,const std::string&);
00271     void createTriMesh(
00272       const std::string &meshName,
00273       const MeshNodeDataVector &mndVector,
00274       const MeshTriDataVector &mtdVector
00275     );    
00276     void addMesh2D(const std::string&,const std::string&,int);
00277     void addMesh2DIG(const ETriMeshIP &prms);
00278     void addTriMesh(const std::string &meshName, const std::string &fileName);
00279     void addTriMeshIG(const ETriMeshIP &prms);
00280     
00281     void addBondedTriMeshIG(const BTriMeshIP &triMeshPrms, const MeshTagBuildPrms &buildPrms);
00282     void addBondedTriMeshIG(const BTriMeshIP &triMeshPrms, const MeshGapBuildPrms &buildPrms);
00283   
00284     void addBondedMesh2DIG(const BMesh2DIP&, const MeshTagBuildPrms&){std::cerr << "NOT IMPLEMENTED" << endl;}; 
00285     void addBondedMesh2DIG(const BMesh2DIP&, const MeshGapBuildPrms&); 
00286     void addDamping(const CDampingIGP &dampingIGP);
00287     void addDamping(const CLocalDampingIGP &dampingIGP);
00288     void addDamping(const ABCDampingIGP &dampingIGP);
00289     
00290     void addSingleIG(const esys::lsm::GravityIGP &gravityIGP);
00291     void addExIG(const std::string&,const std::string&);
00292     void setNumSteps(int s);
00293     int getNumSteps() const {return m_max_ts;};
00294     int getSteps() const {return m_t;};
00295 
00296     void addRotBondedIG(int,const std::string&,double,double,double,double,double,double,double,double,bool);  
00297     void addRotThermBondedIG(const CRotThermBondedIGP &prms);
00298 
00299 //    ParticleIdPairVector getBondGroupIdPairs(const std::string &groupName);
00300 
00301     // --- wall related fucntions ---  
00302     void addWall(const std::string&,const Vec3&,const Vec3&);
00303     void addWallIG(const CEWallIGP&);
00304     void addWallIG(const CBWallIGP&);
00305     void addWallIG(const CVWallIGP&);
00306     void addWallIG(const CSoftBWallIGP&);
00307     Vec3 getWallPosn(const std::string&);
00308     Vec3 getWallForce(const std::string&);
00309     
00310     //     void initSoftBondedWall(const string&,const Vec3&,const Vec3&,double,double,double,int);
00311  
00312 
00313     void moveParticleTo(int particleTag, const Vec3 &posn);
00314     void moveTaggedParticlesBy(int particleTag, const Vec3 &displacement);
00315     void moveSingleParticleTo(int particleId, const Vec3 &posn);
00316     Vec3 getParticlePosn(int particleId);
00317     void setParticleNonDynamic(int);
00318     void setParticleNonRot(int);
00319     void setParticleNonTrans(int);
00320     void setParticleVel(int,const Vec3&);
00321     void setParticleAngVel(int,const Vec3&);
00322     void setParticleDensity(int tag,int mask,double rho);
00323     void setTaggedParticleVel(int tag,const Vec3&);
00324     void moveWallBy(const std::string&,const Vec3&);
00325     void setWallNormal(const std::string&,const Vec3&);
00326     void setVelocityOfWall(const std::string&,const Vec3&);
00327     void tagParticleNearestTo(int,int,const Vec3&);
00332     int findParticleNearestTo(const Vec3& pos);
00333     void applyForceToWall(const std::string&,const Vec3&);
00334     // --- Mesh movement functions ---
00335     void moveSingleNodeBy(const std::string&,int,const Vec3&);
00336     void moveTaggedNodesBy(const std::string&,int,const Vec3&);
00337     void translateMeshBy(const std::string&,const Vec3&);
00338 
00339     void saveTimingDataToFile(const std::string &fileNamePrefix);
00340 
00344     void do2dCalculations(bool do2d);
00345 
00355     void setProcessDims(const esys::lsm::CLatticeParam::ProcessDims &dims);
00356 
00357     const esys::lsm::CLatticeParam::ProcessDims &getProcessDims() const;
00358 
00364     void setTimingFileName(const std::string &fileName);
00365 
00372     const std::string &getTimingFileName() const;
00373     
00374     const std::string &getParticleType() const
00375     {
00376       return m_particle_type;
00377     }
00378 
00379     int getNumParticles();
00380 
00382     void addScalarParticleSaveField(const std::string&,const std::string&,const std::string&,int,int,int);
00383     void addTaggedScalarParticleSaveField(const std::string&,const std::string&,const std::string&,int,int,int,int,int);
00384     void addVectorParticleSaveField(const std::string&,const std::string&,const std::string&,int,int,int);
00385     void addTaggedVectorParticleSaveField(const std::string&,const std::string&,const std::string&,int,int,int,int,int);
00386     void addScalarInteractionSaveField(const std::string&,const std::string&,const std::string&,const std::string&,const std::string&,int,int,int,bool checked=false);
00387     void addVectorInteractionSaveField(const std::string&,const std::string&,const std::string&,const std::string&,const std::string&,int,int,int,bool checked=false);
00388     void addTaggedScalarInteractionSaveField(const std::string&,const std::string&,const std::string&,const std::string&,const std::string&,int,int,int,int,int,bool);
00389     void addTaggedScalarParticleDistributionSaver(const std::string&,const std::string&,const std::string&,int,int,int,int,int,int,double,double,int);
00390     void addVectorTriangleSaveField(const string&,const string&,const string&,const string&,int,int,int);
00391     void addScalarTriangleSaveField(const string&,const string&,const string&,const string&,int,int,int);
00392     void addVectorWallField(const string&,const string&,vector<string>,const string&,int,int,int);
00393     // fields with trigger
00394     void addVectorParticleSaveFieldWT(const std::string&,const std::string&,const std::string&,int,int,int,const MaxTrigParams&);
00395     void addTaggedVectorParticleSaveFieldWT(const std::string&,const std::string&,const std::string&,int,int,int,int,int,const MaxTrigParams&);
00396 
00406     void performCheckPoints(
00407         const std::string &fileNamePrefix,
00408         int beginTime,
00409         int endTime,
00410         int timeInterval
00411     );
00412 
00424     void performCheckPointsThroughMaster(
00425         const std::string &fileNamePrefix,
00426         int beginTime,
00427         int endTime,
00428         int timeInterval
00429     );
00430 
00431     void initSnapShotController(const std::string&,int,int,int);
00432 
00434     void makeLattice(
00435       const char *particleType,
00436       double gridSize,
00437       double verletDist
00438     );
00439 
00440     void makeLattice(
00441       const char *particleType,
00442       double gridSize,
00443       double verletDist,
00444       double dt
00445     );
00446 
00451     void addPreTimeStepRunnable(esys::lsm::Runnable &runnable);
00452 
00456     const RunnableVector &getPreTimeStepRunnableVector() const
00457     {
00458       return m_preRunnableVector;
00459     }
00460 
00464     RunnableVector &getPreTimeStepRunnableVector()
00465     {
00466       return m_preRunnableVector;
00467     }
00468 
00473     void addPostTimeStepRunnable(esys::lsm::Runnable &runnable);
00474 
00478     const RunnableVector &getPostTimeStepRunnableVector() const
00479     {
00480       return m_postRunnableVector;
00481     }
00482 
00486     RunnableVector &getPostTimeStepRunnableVector()
00487     {
00488       return m_postRunnableVector;
00489     }
00490 
00494     void setSpatialDomain(const Vec3 &minBBoxPt, const Vec3 &maxBBoxPt);
00495     
00499     void setSpatialDomain(
00500       const Vec3 &minBBoxPt,
00501       const Vec3 &maxBBoxPt,
00502       const esys::lsm::IntVector &circDimVector
00503     );
00504 
00509     bool haveSetSpatialDomain() const;
00510 
00514     void getSlaveSpatialDomains();
00515 
00522     void initialiseGeometry(const esys::lsm::GeometryInfo &geoInfo);
00523     void initialiseGeometry(const esys::lsm::GeometryInfo &geoInfo,bool);
00531     template <class TmplParticle>
00532     void  readGeometry(const std::string &fileName);
00533 
00539     void readGeometryFile(const std::string &fileName);
00540 
00548     void loadCheckPointData(const std::string &checkPointFileName);
00549 
00558     template <class TmplIterator, class TmplParticle>
00559     void addParticles(TmplIterator &it);
00560 
00569     template <class TmplIterator>
00570     void addConnections(TmplIterator &it);
00571 
00572     //--- function for mesh data exchange ---
00573     template <typename TmplVisitor>
00574     void visitMeshFaceReferences(const string &meshName);
00575 
00576     template <typename TmplVisitor>
00577     void visitMesh2dNodeReferences(const string &meshName, TmplVisitor &visitor);
00578 
00579     template <typename TmplVisitor>
00580     void visitMesh2dEdgeStress(const string &meshName, TmplVisitor &visitor);
00581 
00582     template <typename TmplVisitor>
00583     void visitTriMeshFaceForce(
00584       const string &meshName,
00585       TmplVisitor &visitor
00586     );
00587 
00588     typedef std::vector<int> IdVector;
00589 
00590     template <typename TmplVisitor, typename TmplParticle>
00591     void visitParticlesOfType(
00592       const IdVector &particleIdVector,
00593       TmplVisitor &visitor
00594     );
00595 
00596     template <typename TmplVisitor>
00597     void visitParticles(const IdVector &particleIdVector, TmplVisitor &visitor);
00598 
00599     void setVerbosity(bool);
00600 };
00601 
00602 #include "Parallel/LatticeMaster.hpp"
00603 
00604 #endif