[std0153:5929 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.09 seconds) [std0153:5921 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.15 seconds) [std0153:5923 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.18 seconds) [std0153:5927 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.21 seconds) [std0153:5926 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.25 seconds) [std0153:5928 :0] ud_channel.c:768 Fatal: UD timeout sending to std0171:5931 (after 600.28 seconds) ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5929 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9] ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5921 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9] ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5923 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9] ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5927 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9] ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5926 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9] ==== backtrace ==== 2 0x000000000005a42c mxm_handle_error() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/debug.c:641 3 0x000000000005b1ae __mxm_abort() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/util/debug/log.c:257 4 0x0000000000063d5f mxm_ud_channel_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_channel.c:766 5 0x00000000000676b5 mxm_ud_ep_timer() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/tl/ud/ud_ep.c:144 6 0x000000000006006c mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 7 0x00000000000452bb mxm_timerq_sweep() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/time/timerq.h:71 8 0x0000000000041df0 mxm_invoke_callback() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/./mxm/util/datatype/callback.h:64 9 0x0000000000041df0 mxm_async_missed() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.c:818 10 0x0000000000045543 mxm_async_check_miss() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/async.h:222 11 0x0000000000045543 mxm_progress_internal() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:65 12 0x0000000000045543 mxm_progress() /var/tmp/OFED_topdir/BUILD/mxm-3.5.3092/src/mxm/core/mxm.c:346 13 0x0000000000263744 mca_pml_yalla_progress() ??:0 14 0x000000000002e20b opal_progress() ??:0 15 0x000000000008b002 ompi_request_default_wait_all() ??:0 16 0x00000000000c5d43 PMPI_Waitall() ??:0 17 0x000000000000750f _ZN4Foam8UPstream12waitRequestsEi() ??:0 18 0x00000000004e4e93 _ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h() ??:0 19 0x00000000004f5c57 _ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi() ??:0 20 0x00000000004f609a _ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi() ??:0 21 0x0000000000513d9c _ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h() ??:0 22 0x00000000005158ff _ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h() ??:0 23 0x0000000000b794ad _ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE() ??:0 24 0x0000000000462463 _ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE() ??:0 25 0x0000000000462771 _ZN4Foam8fvMatrixIdE5solveEv() ??:0 26 0x0000000000424290 main() ??:0 27 0x000000000001ed20 __libc_start_main() ??:0 28 0x0000000000425bd9 _start() ??:0 =================== pisoFoam:5928 terminated with signal 6 at PC=2aaaaff234f5 SP=7fffffff77a8. Backtrace: /lib64/libc.so.6(gsignal+0x35)[0x2aaaaff234f5] /lib64/libc.so.6(abort+0x175)[0x2aaaaff24cd5] /app/mellanox/mxm/lib/libmxm.so.2(+0x5b1b3)[0x2aaab6e711b3] /app/mellanox/mxm/lib/libmxm.so.2(mxm_ud_channel_timer+0x42f)[0x2aaab6e79d5f] /app/mellanox/mxm/lib/libmxm.so.2(+0x676b5)[0x2aaab6e7d6b5] /app/mellanox/mxm/lib/libmxm.so.2(mxm_timerq_sweep_internal+0x6c)[0x2aaab6e7606c] /app/mellanox/mxm/lib/libmxm.so.2(+0x452bb)[0x2aaab6e5b2bb] /app/mellanox/mxm/lib/libmxm.so.2(mxm_async_missed+0x110)[0x2aaab6e57df0] /app/mellanox/mxm/lib/libmxm.so.2(mxm_progress+0x133)[0x2aaab6e5b543] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(mca_pml_yalla_progress+0x14)[0x2aaab2f7f744] /app/openmpi/1.10.2/xe_2016/lib/libopen-pal.so.13(opal_progress+0xab)[0x2aaab76f220b] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(ompi_request_default_wait_all+0x352)[0x2aaab2da7002] /app/openmpi/1.10.2/xe_2016/lib/libmpi.so.12(PMPI_Waitall+0x83)[0x2aaab2de1d43] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/openmpi-system/libPstream.so(_ZN4Foam8UPstream12waitRequestsEi+0xdf)[0x2aaab028c50f] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam9lduMatrix22updateMatrixInterfacesERKNS_10FieldFieldINS_5FieldEdEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEERKNS2_IdEERSC_h+0x5f3)[0x2aaaaee70e93] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZN4Foam19GaussSeidelSmoother6smoothERKNS_4wordERNS_5FieldIdEERKNS_9lduMatrixERKS5_RKNS_10FieldFieldIS4_dEERKNS_8UPtrListIKNS_17lduInterfaceFieldEEEhi+0x287)[0x2aaaaee81c57] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam19GaussSeidelSmoother6smoothERNS_5FieldIdEERKS2_hi+0x2a)[0x2aaaaee8209a] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver6VcycleERKNS_7PtrListINS_9lduMatrix8smootherEEERNS_5FieldIdEERKS8_S9_S9_S9_S9_S9_RNS1_IS8_EESD_h+0x111c)[0x2aaaaee9fd9c] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libOpenFOAM.so(_ZNK4Foam10GAMGSolver5solveERNS_5FieldIdEERKS2_h+0x44f)[0x2aaaaeea18ff] /home/users/nus/e0920926/OpenFOAM/OpenFOAM-8/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x13d)[0x2aaaac4184ad] pisoFoam(_ZN4Foam8fvMatrixIdE5solveERKNS_10dictionaryE+0x113)[0x462463] pisoFoam(_ZN4Foam8fvMatrixIdE5solveEv+0xf1)[0x462771] pisoFoam[0x424290] /lib64/libc.so.6(__libc_start_main+0x100)[0x2aaaaff0fd20] pisoFoam[0x425bd9]