modified for petsc-complex run (see readme for details)

wrtobin · Aug 11, 2017 · 8f0dc5c · 8f0dc5c
1 parent edf88c5
commit 8f0dc5c
Show file tree

Hide file tree

Showing 9 changed files with 394 additions and 341 deletions.
diff --git a/api/msi.cc b/api/msi.cc
diff --git a/api/msi.h b/api/msi.h
@@ -80,28 +80,28 @@ int msi_matrix_print(int* matrix_id);
 //=========================================================================
 /** matrix and solver functions with TRILINOS */
 //=========================================================================
+int msi_epetra_create(int* matrix_id, int* matrix_type, int* scalar_type, FieldID* field_id);
+int msi_epetra_delete(int* matrix_id);
 
-int msi_epetra_create(int matrix_id, int matrix_type, int field_id);
-int msi_epetra_delete(int matrix_id);
+int msi_epetra_insert(int* matrix_id, int* row, int* column, int* scalar_type, double* val);
+int msi_epetra_addblock(int* matrix_id, int * ielm, int* rowVarIdx, int * columnVarIdx, double * values);
 
-int msi_epetra_insert(int matrix_id, int row, int column, double* val);
-int msi_epetra_addblock(int matrix_id, int * ielm, int rowVarIdx, int * columnVarIdx, double * values);
-
-int msi_epetra_setbc(int matrix_id, int row);
+int msi_epetra_setbc(int* matrix_id, int* row);
 int msi_epetra_setlaplacebc (int * matrix_id, int *row, int * numVals, int *columns, double * values);
-int msi_epetra_freeze(int matrix_id); 
-int msi_epetra_multiply(int matrix_id, int in_fieldid, int out_fieldid);
-int msi_epetra_write(int matrix_id, const char*, int skip_zero, int start_index);
-int msi_epetra_print(int matrix_id);
+int msi_epetra_freeze(int* matrix_id); 
+int msi_epetra_multiply(int* matrix_id, FieldID* in_fieldid, FieldID* out_fieldid);
+int msi_epetra_write(int* matrix_id, const char*, int* skip_zero, int* start_index);
+int msi_epetra_print(int* matrix_id);
 
-int msi_solver_aztec(int matrix_id, int x_fieldid, int
-		       b_fieldid, int num_iter, double* tolerance,
+int msi_solver_aztec(int* matrix_id, FieldID* x_fieldid, FieldID*
+		       b_fieldid, int* num_iter, double* tolerance,
 		       const char* krylov_solver, const char*
 		       preconditioner, const char* sub_dom_solver,
-		       int overlap, int graph_fill, double*
+		       int* overlap, int* graph_fill, double*
 		       ilu_drop_tol,  double* ilu_fill,
-		       double* ilu_omega, int poly_ord);
+		       double* ilu_omega, int* poly_ord);
 
-int msi_solver_getnumiter(int matrix_id, int * iter_num);
+int msi_solver_amesos(int* matrix_id, FieldID* in_fieldid, FieldID* out_fieldid, const char* solver_name);
+int msi_solver_getnumiter(int* matrix_id, int * iter_num);
 #endif //#ifdef MSI_TRILINOS
 #endif
diff --git a/include/msi_trilinos.h b/include/msi_trilinos.h
@@ -1,19 +1,17 @@
 /****************************************************************************** 
 
-  (c) 2005-2017 Scientific Computation Research Center, 
+  (c) 2017 Scientific Computation Research Center, 
       Rensselaer Polytechnic Institute. All rights reserved.
   
   This work is open source software, licensed under the terms of the
   BSD license as described in the LICENSE file in the top-level directory.
  
 *******************************************************************************/
-#ifdef M3DC1_TRILINOS
-#ifndef M3DC1_LS_H
-#define M3DC1_LS_H
+#ifdef MSI_TRILINOS
+#ifndef MSI_LS_H
+#define MSI_LS_H
 #include <map>
 #include "apf.h"
-#include "m3dc1_scorec.h"
-
 #include <Epetra_Map.h>
 #include <Epetra_CrsMatrix.h>
 #ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
@@ -24,12 +22,12 @@ typedef int global_ordinal_type;
 
 void write_matrix(Epetra_CrsMatrix*, const char* filename, bool nozero, int start_index);
 
-// NOTE: all field realted interaction is done through m3dc1 api rather than apf
-class m3dc1_epetra
+// NOTE: all field realted interaction is done through msi api rather than apf
+class msi_epetra
 {
 public:
-  m3dc1_epetra(int i, int t, int s, FieldID field);
-  ~m3dc1_epetra();
+  msi_epetra(int i, int t, int s, FieldID field);
+  ~msi_epetra();
   void destroy(); // delete a matrix and solver object
   int get_scalar_type() { return scalar_type; }
   int get_field_id() { return field_id;}
@@ -46,19 +44,19 @@ class m3dc1_epetra
   int field_id; // the field that provides numbering
 };
 
-class m3dc1_ls
+class msi_ls
 {
 public:
 // functions
-  m3dc1_ls(){matrix_container = new std::map<int, m3dc1_epetra*>;}
-  ~m3dc1_ls();
-  static m3dc1_ls* instance();
-  m3dc1_epetra* get_matrix(int matrix_id);
-  void add_matrix(int matrix_id, m3dc1_epetra*);
+  msi_ls(){matrix_container = new std::map<int, msi_epetra*>;}
+  ~msi_ls();
+  static msi_ls* instance();
+  msi_epetra* get_matrix(int matrix_id);
+  void add_matrix(int matrix_id, msi_epetra*);
 // data
-  std::map<int, m3dc1_epetra*>* matrix_container;
+  std::map<int, msi_epetra*>* matrix_container;
 private:
-  static m3dc1_ls* _instance;
+  static msi_ls* _instance;
 };
 #endif
 #endif
diff --git a/openmpi-gcc4.4.5-complex-config.sh b/openmpi-gcc4.4.5-complex-config.sh
@@ -1,5 +1,5 @@
-PETSC_DIR=/fasttmp/seol/petsc-3.5.4-complex
-PETSC_ARCH=openmpi1.6.5
+PETSC_DIR=/lore/seol/petsc-3.5.4
+PETSC_ARCH=complex-openmpi1.6.5
 PREFIX=/fasttmp/seol/openmpi-gcc4.4.5-install
 cmake .. \
   -DCMAKE_C_COMPILER="/usr/local/openmpi/latest/bin/mpicc" \
@@ -10,7 +10,7 @@ cmake .. \
   -DCMAKE_Fortran_FLAGS="-fpic "\
   -DSCOREC_INCLUDE_DIR=$PREFIX/include \
   -DSCOREC_LIB_DIR=$PREFIXl/lib \
-  -DZOLTAN_LIBRARY="$PETSC_DIR/$PETSC_ARCH/lib/libzoltan.a" \
+  -DZOLTAN_LIBRARY="$PREFIX/lib/libzoltan.a" \
   -DPARMETIS_LIBRARY="$PETSC_DIR/$PETSC_ARCH/lib/libparmetis.a" \
   -DMETIS_LIBRARY="$PETSC_DIR/$PETSC_ARCH/lib/libmetis.a" \
   -DENABLE_SIMMETRIX=OFF \

diff --git a/readme b/readme
@@ -11,9 +11,16 @@ To compile MSI test program with petsc located in test/petsc:
 =============================
 export LD_LIBRARY_PATH=:/usr/local/openmpi/latest/lib:/fasttmp/seol/openmpi-gcc4.4.5-install/lib:/lore/seol/petsc-3.5.4/real-openmpi1.6.5/lib:/usr/lib/gcc/x86_64-linux-gnu/4.4.5:/usr/lib/x86_64-linux-gnu:
 
+#real
 /usr/local/openmpi/latest/bin/mpicc ../test/petsc/main.cc -o petsc -DDEBUG -DMSI_PETSC -I/usr/local/openmpi/latest/include  -I/lore/seol/petsc-3.5.4/real-openmpi1.6.5/include -I/lore/seol/petsc-3.5.4/include -I/fasttmp/seol/openmpi-gcc4.4.5-install/include -Wl,--start-group,-rpath,/fasttmp/seol/openmpi-gcc4.4.5-install/lib -L/fasttmp/seol/openmpi-gcc4.4.5-install/lib -lmsi -lpumi -lcrv -ldsp -lph -lsize -lsam -lspr -lma -lapf_zoltan -lparma -lmds -lapf -llion -lmth -lgmi -lpcu -lzoltan -Wl,--end-group -L/lore/seol/petsc-3.5.4/real-openmpi1.6.5/lib  -lpetsc -Wl,-rpath,/lore/seol/petsc-3.5.4/real-openmpi1.6.5/lib -lcmumps -ldmumps -lsmumps -lzmumps -lmumps_common -lpord -lscalapack -lHYPRE -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -lmpi_cxx -lstdc++ -lsuperlu_4.3 -lsuperlu_dist_3.3 -lflapack -lfblas -lparmetis -lmetis -lpthread -lssl -lcrypto -lnetcdf -lhdf5hl_fortran -lhdf5_fortran -lhdf5_hl -lhdf5 -lz -lmpi_f90 -lmpi_f77 -lgfortran -lm -lmpi_cxx -lstdc++ -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -lmpi_cxx -lstdc++ -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -ldl -lmpi -lnuma -lrt -lnsl -lutil -lgcc_s -lpthread -ldl  
 
+#complex
+/usr/local/openmpi/latest/bin/mpicc ../test/petsc/main.cc -o petsc -DDEBUG -DMSI_PETSC -DPETSC_USE_COMPLEX -I/usr/local/openmpi/latest/include  -I/lore/seol/petsc-3.5.4/complex-openmpi1.6.5/include -I/lore/seol/petsc-3.5.4/include -I/fasttmp/seol/openmpi-gcc4.4.5-install/include -Wl,--start-group,-rpath,/fasttmp/seol/openmpi-gcc4.4.5-install/lib -L/fasttmp/seol/openmpi-gcc4.4.5-install/lib -lmsi_complex -lpumi -lcrv -ldsp -lph -lsize -lsam -lspr -lma -lapf_zoltan -lparma -lmds -lapf -llion -lmth -lgmi -lpcu -lzoltan -Wl,--end-group -L/lore/seol/petsc-3.5.4/complex-openmpi1.6.5/lib  -lpetsc -Wl,-rpath,/lore/seol/petsc-3.5.4/complex-openmpi1.6.5/lib -lcmumps -ldmumps -lsmumps -lzmumps -lmumps_common -lpord -lscalapack -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -lmpi_cxx -lstdc++ -lsuperlu_4.3 -lsuperlu_dist_3.3 -lflapack -lfblas -lparmetis -lmetis -lpthread -lssl -lcrypto -lnetcdf -lhdf5hl_fortran -lhdf5_fortran -lhdf5_hl -lhdf5 -lz -lmpi_f90 -lmpi_f77 -lgfortran -lm -lmpi_cxx -lstdc++ -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -lmpi_cxx -lstdc++ -L/usr/local/openmpi/1.6.5-ib/lib -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/gcc/x86_64-linux-gnu/4.4.5 -L/usr/lib/x86_64-linux-gnu -ldl -lmpi -lnuma -lrt -lnsl -lutil -lgcc_s -lpthread -ldl
+
 =============================
 To run MSI test program with petsc
 =============================
-/usr/local/openmpi/latest/bin/mpirun -np 4 ./petsc /lore/seol/meshes/1K-4part/model.dmg /lore/seol/meshes/1K-4part/part.smb out.smb 
+./petsc model(.dmg) distributed-mesh(.smb)
+
+<Example>
+/usr/local/openmpi/latest/bin/mpirun -np 4 ./petsc /lore/seol/meshes/1K-4part/model.dmg /lore/seol/meshes/1K-4part/part.smb