Running on Solaris/SPARC, getting frequent crashes, that *usually* do not happen twice in a row - meaning, I run it once, it crashes, change nothing, run again, run successfully.
Here is some GDB backtraces from the core-dump, anywhere you <...snip...> I've removed data to protect the ... innocent?
Let me know if this is helpful or not and if you need more info.
[code1dc46700bfc]
(gdb) core-file core-20050517123503
Core was generated by `/u01/app/netmrg/bin/netmrg-gatherer -a -C /u01/app/netmrg/etc/netmrg.xml'.
Program terminated with signal 10, Bus error.
Reading symbols from /usr/local/lib/libnetsnmp.so.5...done.
Loaded symbols for /usr/local/lib/libnetsnmp.so.5
Reading symbols from /usr/local/ssl/lib/libcrypto.so.0.9.7...done.
Loaded symbols for /usr/local/ssl/lib/libcrypto.so.0.9.7
Reading symbols from /usr/lib/librt.so.1...done.
Loaded symbols for /usr/lib/librt.so.1
Reading symbols from /u01/app/mysql/lib/mysql/libmysqlclient.so.12...done.
Loaded symbols for /u01/app/mysql/lib/mysql/libmysqlclient.so.12
Reading symbols from /usr/local/lib/libxml2.so.2...done.
Loaded symbols for /usr/local/lib/libxml2.so.2
Reading symbols from /usr/local/lib/libstdc++.so.6...done.
Loaded symbols for /usr/local/lib/libstdc++.so.6
Reading symbols from /usr/lib/libpthread.so.1...done.
Loaded symbols for /usr/lib/libpthread.so.1
Reading symbols from /usr/local/lib/libz.so...done.
Loaded symbols for /usr/local/lib/libz.so
Reading symbols from /usr/lib/libdl.so.1...done.
Loaded symbols for /usr/lib/libdl.so.1
Reading symbols from /usr/lib/libintl.so.1...warning: Lowest section in /usr/lib/libintl.so.1 is .hash at 00000074
done.
Loaded symbols for /usr/lib/libintl.so.1
Reading symbols from /usr/lib/libkstat.so.1...done.
Loaded symbols for /usr/lib/libkstat.so.1
Reading symbols from /usr/lib/libm.so.1...done.
Loaded symbols for /usr/lib/libm.so.1
Reading symbols from /usr/local/lib/libgcc_s.so.1...done.
Loaded symbols for /usr/local/lib/libgcc_s.so.1
Reading symbols from /usr/lib/libc.so.1...done.
Loaded symbols for /usr/lib/libc.so.1
Reading symbols from /usr/lib/libsocket.so.1...done.
Loaded symbols for /usr/lib/libsocket.so.1
Reading symbols from /usr/lib/libnsl.so.1...done.
Loaded symbols for /usr/lib/libnsl.so.1
Reading symbols from /usr/lib/libaio.so.1...done.
Loaded symbols for /usr/lib/libaio.so.1
Reading symbols from /usr/lib/libcrypt_i.so.1...done.
Loaded symbols for /usr/lib/libcrypt_i.so.1
Reading symbols from /usr/lib/libgen.so.1...done.
Loaded symbols for /usr/lib/libgen.so.1
Reading symbols from /usr/local/lib/libiconv.so.2...done.
Loaded symbols for /usr/local/lib/libiconv.so.2
Reading symbols from /usr/lib/libmp.so.2...done.
Loaded symbols for /usr/lib/libmp.so.2
Reading symbols from /usr/platform/SUNW,Ultra-4/lib/libc_psr.so.1...done.
Loaded symbols for /usr/platform/SUNW,Ultra-4/lib/libc_psr.so.1
Reading symbols from /usr/lib/libthread.so.1...done.
Loaded symbols for /usr/lib/libthread.so.1
Reading symbols from /usr/lib/nss_files.so.1...done.
Loaded symbols for /usr/lib/nss_files.so.1
#0 std::string::append(std::string const&) (this=0xfe509618, __str=@0x27bf08) at basic_string.h:265
265 basic_string.h: No such file or directory.
in basic_string.h
(gdb) bt
#0 std::string::append(std::string const&) (this=0xfe509618, __str=@0x27bf08) at basic_string.h:265
#1 0x0003b55c in std::basic_string<char, std::char_traits<char>, std::allocator<char> > std::operator+<char, std::char_traits<char>, std::allocator<char> >(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (__lhs=@0xfe509610, __rhs=@0x27bf08)
at bits/basic_string.h:1985
#2 0x00015608 in process_sub_device(DeviceInfo, st_mysql*) (info=
{device_id = 440796, subdevice_id = 4, monitor_id = 440796, event_id = -17491376, response_id = 2819380, status = 2672430, mysql = 0x21, snmp_sess_p = 0x7efefeff, snmp_avoid = 2819132, snmp_recache = 65280, snmp_ifnumber = 4266694256, snmp_uptime = 1901175863750292, subdevice_type = 2224364, test_type = 2325260, test_id = -28273016, name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xff248b80 "@"}}, ip = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6bb44 "INSE"}}, subdevice_name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfe509708 ""}}, test_params = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x40df8 "] "}}, curr_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xf3333333 <Address 0xf3333333 out of bounds>}}, last_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x4 <Address 0x4 out of bounds>}}, delta_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfe509780 ""}}, rate_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfe5096d0 ""}}, delta_time = 404480, snmp_version = 0, snmp_read_community = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x29c758 "<snip>"}}, snmp_timeout = 4266694496, snmp_retries = 4294967295, snmp_port = 65104, parameters = {<_List_base<ValuePair,std::allocator<ValuePair> >> = {_M_impl = {<allocator<std::_List_node<ValuePair> >> = {<new_allocator<std::_List_node<ValuePair> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0xfe509a98, _M_prev = 0xfe509768}}}, <No data fields>}}, mysql=0x27bf08) at bits/stl_list.h:134
[/code1dc46700bfc]
[code1dc46700bfc]
(gdb) core-file core-20050517114004
Core was generated by `/u01/app/netmrg/bin/netmrg-gatherer -a -C /u01/app/netmrg/etc/netmrg.xml'.
Program terminated with signal 10, Bus error.
Loaded symbols for /usr/local/lib/libnetsnmp.so.5
Loaded symbols for /usr/local/ssl/lib/libcrypto.so.0.9.7
Loaded symbols for /usr/lib/librt.so.1
Reading symbols from /u01/app/mysql/lib/mysql/libmysqlclient.so.12...done.
Loaded symbols for /u01/app/mysql/lib/mysql/libmysqlclient.so.12
Loaded symbols for /usr/local/lib/libxml2.so.2
Loaded symbols for /usr/local/lib/libstdc++.so.6
Loaded symbols for /usr/lib/libpthread.so.1
Loaded symbols for /usr/local/lib/libz.so
Loaded symbols for /usr/lib/libdl.so.1
Loaded symbols for /usr/lib/libintl.so.1
Loaded symbols for /usr/lib/libkstat.so.1
Loaded symbols for /usr/lib/libm.so.1
Loaded symbols for /usr/local/lib/libgcc_s.so.1
Loaded symbols for /usr/lib/libc.so.1
Loaded symbols for /usr/lib/libsocket.so.1
Loaded symbols for /usr/lib/libnsl.so.1
Loaded symbols for /usr/lib/libaio.so.1
Loaded symbols for /usr/lib/libcrypt_i.so.1
Loaded symbols for /usr/lib/libgen.so.1
Loaded symbols for /usr/local/lib/libiconv.so.2
Loaded symbols for /usr/lib/libmp.so.2
Loaded symbols for /usr/platform/SUNW,Ultra-4/lib/libc_psr.so.1
Loaded symbols for /usr/lib/libthread.so.1
Loaded symbols for /usr/lib/nss_files.so.1
#0 basic_string (this=0x2c9400, __str=@0x232306) at basic_string.h:182
182 in basic_string.h
(gdb) bt
#0 basic_string (this=0x2c9400, __str=@0x232306) at basic_string.h:182
#1 0x0003c134 in DeviceInfo (this=0xfe4076d0, _ctor_arg=@0xfe40788c) at devices.cpp:28
#2 0x00017060 in process_sub_devices(DeviceInfo, st_mysql*) (info=
{device_id = 5, subdevice_id = 410, monitor_id = -1, event_id = -1, response_id = -1, status = 0, mysql = 0xfe407a98, snmp_sess_p = 0x27f510, snmp_avoid = 0, snmp_recache = 0, snmp_ifnumber = 8, snmp_uptime = 137181796, subdevice_type = 2, test_type = -1, test_id = -1, name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x23464c "<...snip...>"}}, ip = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x22550c "<...snip...>"}}, subdevice_name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x2c9e04 "<...snip...>"}}, test_params = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x62f0c ""}}, curr_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27f2d4 "U"}}, last_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27f274 "U"}}, delta_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27fa1c "U"}}, rate_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27fa34 "U"}}, delta_time = 0, snmp_version = 2, snmp_read_community = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x2306a4 "<...snip...>"}}, snmp_timeout = 1000000, snmp_retries = 3, snmp_port = 161, parameters = {<_List_base<ValuePair,std::allocator<ValuePair> >> = {_M_impl = {<allocator<std::_List_node<ValuePair> >> = {<new_allocator<std::_List_node<ValuePair> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0xfe407754, _M_prev = 0xfe407754}}}, <No data fields>}}, mysql=0xfe40788c) at devices.cpp:175
[/code1dc46700bfc]
[code1dc46700bfc]
(gdb) core-file core-20050517104504
Core was generated by `/u01/app/netmrg/bin/netmrg-gatherer -a -C /u01/app/netmrg/etc/netmrg.xml'.
Program terminated with signal 10, Bus error.
Loaded symbols for /usr/local/lib/libnetsnmp.so.5
Loaded symbols for /usr/local/ssl/lib/libcrypto.so.0.9.7
Loaded symbols for /usr/lib/librt.so.1
Reading symbols from /u01/app/mysql/lib/mysql/libmysqlclient.so.12...done.
Loaded symbols for /u01/app/mysql/lib/mysql/libmysqlclient.so.12
Loaded symbols for /usr/local/lib/libxml2.so.2
Loaded symbols for /usr/local/lib/libstdc++.so.6
Loaded symbols for /usr/lib/libpthread.so.1
Loaded symbols for /usr/local/lib/libz.so
Loaded symbols for /usr/lib/libdl.so.1
Loaded symbols for /usr/lib/libintl.so.1
Loaded symbols for /usr/lib/libkstat.so.1
Loaded symbols for /usr/lib/libm.so.1
Loaded symbols for /usr/local/lib/libgcc_s.so.1
Loaded symbols for /usr/lib/libc.so.1
Loaded symbols for /usr/lib/libsocket.so.1
Loaded symbols for /usr/lib/libnsl.so.1
Loaded symbols for /usr/lib/libaio.so.1
Loaded symbols for /usr/lib/libcrypt_i.so.1
Loaded symbols for /usr/lib/libgen.so.1
Loaded symbols for /usr/local/lib/libiconv.so.2
Loaded symbols for /usr/lib/libmp.so.2
Loaded symbols for /usr/platform/SUNW,Ultra-4/lib/libc_psr.so.1
Loaded symbols for /usr/lib/libthread.so.1
Loaded symbols for /usr/lib/nss_files.so.1
#0 basic_string (this=0x2d7fa8, __str=@0x21f52e) at basic_string.h:182
182 in basic_string.h
(gdb) bt
#0 basic_string (this=0x2d7fa8, __str=@0x21f52e) at basic_string.h:182
#1 0x0003c134 in DeviceInfo (this=0xfe5096d0, _ctor_arg=@0xfe50988c) at devices.cpp:28
#2 0x00017060 in process_sub_devices(DeviceInfo, st_mysql*) (info=
{device_id = 4, subdevice_id = 175, monitor_id = -1, event_id = -1, response_id = -1, status = 0, mysql = 0xfe509a98, snmp_sess_p = 0x27f208, snmp_avoid = 0, snmp_recache = 0, snmp_ifnumber = 100, snmp_uptime = 979107209, subdevice_type = 2, test_type = -1, test_id = -1, name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x229fc4 "<...snip...>"}}, ip = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x229de4 "<snip>"}}, subdevice_name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x186294 "<...snip...>"}}, test_params = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x62f0c ""}}, curr_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27d894 "U"}}, last_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27e28c "U"}}, delta_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27bd14 "U"}}, rate_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x27c164 "U"}}, delta_time = 0, snmp_version = 2, snmp_read_community = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x21f6cc "<...snip...>"}}, snmp_timeout = 1000000, snmp_retries = 3, snmp_port = 161, parameters = {<_List_base<ValuePair,std::allocator<ValuePair> >> = {_M_impl = {<allocator<std::_List_node<ValuePair> >> = {<new_allocator<std::_List_node<ValuePair> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0x2bb768, _M_prev = 0x2d7fd0}}}, <No data fields>}}, mysql=0xfe50988c) at devices.cpp:175
[/code1dc46700bfc]
[code1dc46700bfc]
(gdb) core-file core-20050517092026
Core was generated by `/u01/app/netmrg/bin/netmrg-gatherer -a -C /u01/app/netmrg/etc/netmrg.xml'.
Program terminated with signal 10, Bus error.
Loaded symbols for /usr/local/lib/libnetsnmp.so.5
Loaded symbols for /usr/local/ssl/lib/libcrypto.so.0.9.7
Loaded symbols for /usr/lib/librt.so.1
Reading symbols from /u01/app/mysql/lib/mysql/libmysqlclient.so.12...done.
Loaded symbols for /u01/app/mysql/lib/mysql/libmysqlclient.so.12
Loaded symbols for /usr/local/lib/libxml2.so.2
Loaded symbols for /usr/local/lib/libstdc++.so.6
Loaded symbols for /usr/lib/libpthread.so.1
Loaded symbols for /usr/local/lib/libz.so
Loaded symbols for /usr/lib/libdl.so.1
Loaded symbols for /usr/lib/libintl.so.1
Loaded symbols for /usr/lib/libkstat.so.1
Loaded symbols for /usr/lib/libm.so.1
Loaded symbols for /usr/local/lib/libgcc_s.so.1
Loaded symbols for /usr/lib/libc.so.1
Loaded symbols for /usr/lib/libsocket.so.1
Loaded symbols for /usr/lib/libnsl.so.1
Loaded symbols for /usr/lib/libaio.so.1
Loaded symbols for /usr/lib/libcrypt_i.so.1
Loaded symbols for /usr/lib/libgen.so.1
Loaded symbols for /usr/local/lib/libiconv.so.2
Loaded symbols for /usr/lib/libmp.so.2
Loaded symbols for /usr/platform/SUNW,Ultra-4/lib/libc_psr.so.1
Loaded symbols for /usr/lib/libthread.so.1
Loaded symbols for /usr/lib/nss_files.so.1
#0 std::basic_string<char, std::char_traits<char>, std::allocator<char> > std::operator+<char, std::char_traits<char>, std::allocator<char> >(char const*, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
(__lhs=0x3f170 "netmrg_", __rhs=@0x27f4b8) at bits/basic_string.h:265
265 { return &((reinterpret_cast<_Rep*> (_M_data()))[-1]); }
(gdb) bt
#0 std::basic_string<char, std::char_traits<char>, std::allocator<char> > std::operator+<char, std::char_traits<char>, std::allocator<char> >(char const*, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
(__lhs=0x3f170 "netmrg_", __rhs=@0x27f4b8) at bits/basic_string.h:265
#1 0x00023004 in remove_params_from_env(DeviceInfo&) (info=@0xfd501374) at bits/stl_list.h:134
#2 0x00024ed0 in process_script_monitor(DeviceInfo, st_mysql*) (info=
{device_id = 2620280, subdevice_id = 2620304, monitor_id = 65280, event_id = -45083720, response_id = 90164, status = 2921876, mysql = 0x2906a8, snmp_sess_p = 0x22c4f4, snmp_avoid = 2620724, snmp_recache = 0, snmp_ifnumber = 0, snmp_uptime = 19, subdevice_type = 4080218931, test_type = 4, test_id = 29, name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}}, ip = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfd501420 ""}}, subdevice_name = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}}, test_params = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfd501438 ""}}, curr_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfd5014e0 ""}}, last_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}}, delta_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfd5014c8 ""}}, rate_val = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xfd5014cc ""}}, delta_time = -193625543839580160, snmp_version = 0, snmp_read_community = {static npos = 4294967295, _M_dataplus = {<allocator<char>> = {<new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6bbb8 ""}}, snmp_timeout = 404480, snmp_retries = 4249884272, snmp_port = 1, parameters = {<_List_base<ValuePair,std::allocator<ValuePair> >> = {_M_impl = {<allocator<std::_List_node<ValuePair> >> = {<new_allocator<std::_List_node<ValuePair> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0xfd5015f8, _M_prev = 0x6f}}}, <No data fields>}}, mysql=0x62c00) at monitors.cpp:241
[/code1dc46700bfc]