There's something racy in ServerDisconnect.is_detected_by_client

gdb) bt
#0 0x00007f64dbbbd267 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:55
#1 0x00007f64dbbbeeca in __GI_abort () at abort.c:89
#2 0x00007f64dc416b7d in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3 0x00007f64dc4149c6 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00007f64dc414a11 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#5 0x00007f64dc41555f in __cxa_pure_virtual () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6 0x00007f64dd94b0b9 in mir::client::BufferStream::process_buffer (this=0x7f64c8001730, buffer=..., lk=...)
    at /home/alan/display_server/mir2/src/client/buffer_stream.cpp:509
#7 0x00007f64dd94c312 in mir::client::BufferStream::buffer_available (this=0x7f64c8001730, buffer=...)
    at /home/alan/display_server/mir2/src/client/buffer_stream.cpp:709
#8 0x00007f64dd9779a5 in mir::client::rpc::MirProtobufRpcChannel::<lambda(mir::client::ClientBufferStream*)>::operator()(mir::client::ClientBufferStream *) const (__closure=0x7f64d84cf520, stream=0x7f64c8001738) at /home/alan/display_server/mir2/src/client/rpc/mir_protobuf_rpc_channel.cpp:279
#9 0x00007f64dd979035 in std::_Function_handler<void(mir::client::ClientBufferStream*), mir::client::rpc::MirProtobufRpcChannel::process_event_sequence(const string&)::<lambda(mir::client::ClientBufferStream*)> >::_M_invoke(const std::_Any_data &, <unknown type in bin/../lib/libmirclient.so.9, CU 0x37e003, DIE 0x3ad405>) (__functor=..., __args#0=<unknown type in bin/../lib/libmirclient.so.9, CU 0x37e003, DIE 0x3ad405>) at /usr/include/c++/5/functional:1871
#10 0x00007f64dd934545 in std::function<void (mir::client::ClientBufferStream*)>::operator()(mir::client::ClientBufferStream*) const (
    this=0x7f64d84cf520, __args#0=0x7f64c8001738) at /usr/include/c++/5/functional:2271
#11 0x00007f64dd9335c5 in mir::client::ConnectionSurfaceMap::with_stream_do(mir::IntWrapper<mir::frontend::detail::SessionsBufferStreamIdTag, int>, std::function<void (mir::client::ClientBufferStream*)> const&) const (this=0x31485d0, stream_id=..., exec=...)
    at /home/alan/display_server/mir2/src/client/surface_map.cpp:105
#12 0x00007f64dd977d5c in mir::client::rpc::MirProtobufRpcChannel::process_event_sequence (this=0x314b460,
    event="\"\"\n\002\b\000\022\034\b\004\020>\030\357\375\266\365\375\377\377\377\377\001 \001(\274\030\060\000\070\217\006@\263\005")
    at /home/alan/display_server/mir2/src/client/rpc/mir_protobuf_rpc_channel.cpp:280
#13 0x00007f64dd97858b in mir::client::rpc::MirProtobufRpcChannel::on_data_available (this=0x314b460)
    at /home/alan/display_server/mir2/src/client/rpc/mir_protobuf_rpc_channel.cpp:380
#14 0x00007f64dd9703b9 in mir::client::rpc::TransportObservers::<lambda(auto:1)>::operator()<std::shared_ptr<mir::client::rpc::StreamTransport::Observer> >(std::shared_ptr<mir::client::rpc::StreamTransport::Observer>) const (__closure=0x7f64d84cfb10, observer=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<mir::client::rpc::MirProtobufRpcChannel*, mir::client::rpc::MirProtobufRpcChannel::MirProtobufRpcChannel(std::unique_ptr<mir::client::rpc::StreamTranspor...

I *think* the underlying problem is that ConnectionSurfaceMap::with_stream_do() releases the mutex before invoking exec() - which means that another thread can call erase() without blocking and delete the buffer stream.

Hmm, there's a second, rarer (~40000 repeats) crash after the first fix:

(gdb) info threads
  Id Target Id Frame
  3 Thread 0x7f71fa2d1700 (LWP 17905) 0x00007f71fe274a77 in sched_yield () at ../sysdeps/unix/syscall-template.S:81
  2 Thread 0x7f7200cdc7c0 (LWP 30386) 0x00007f71fe55c8ed in pthread_join (threadid=140127513684432, thread_return=0x0) at pthread_join.c:90
* 1 Thread 0x7f71faad2700 (LWP 17904) 0x0000000000000000 in ?? ()
(gdb) bt
#0 0x0000000000000000 in ?? ()
#1 0x00007f71fed2b94f in google::protobuf::MessageLite::ParseFromString(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /usr/lib/x86_64-linux-gnu/libprotobuf-lite.so.9
#2 0x00007f71fff7a7ec in mir::client::rpc::MirProtobufRpcChannel::on_data_available (this=0x1e6b470)
    at /home/alan/display_server/mir2/src/client/rpc/mir_protobuf_rpc_channel.cpp:386
#3 0x00007f71fff72597 in mir::client::rpc::TransportObservers::<lambda(auto:1)>::operator()<std::shared_ptr<mir::client::rpc::StreamTransport::Observer> >(std::shared_ptr<mir::client::rpc::StreamTransport::Observer>) const (__closure=0x7f71faad1b10, observer=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<mir::client::rpc::MirProtobufRpcChannel*, mir::client::rpc::MirProtobufRpcChannel::MirProtobufRpcChannel(std::unique_ptr<mir::client::rpc::StreamTransport, std::default_delete<mir::client::rpc::StreamTransport> >, std::shared_ptr<mir::client::SurfaceMap> const&, std::shared_ptr<mir::client::DisplayConfiguration> const&, std::shared_ptr<mir::client::rpc::RpcReport> const&, std::shared_ptr<mir::client::AtomicCallback<MirLifecycleState> > const&, std::shared_ptr<mir::client::AtomicCallback<int> > const&, std::shared_ptr<mir::client::EventSink> const&)::NullDeleter, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<mir::client::rpc::MirProtobufRpcChannel*, mir::client::rpc::MirProtobufRpcChannel::MirProtobufRpcChannel(std::unique_ptr<mir::client::rpc::StreamTransport, std::default_delete<mir::client::rpc::StreamTransport> >, std::shared_ptr<mir::client::SurfaceMap> const&, std::shared_ptr<mir::client::DisplayConfiguration> const&, std::shared_ptr<mir::client::rpc::RpcReport> const&, std::shared_ptr<mir::client::AtomicCallback<MirLifecycleState> > const&, std::shared_ptr<mir::client::AtomicCallback<int> > const&, std::shared_ptr<mir::client::EventSink> const&)::NullDeleter, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 3, weak 0) 0x1e6b480)
    at /home/alan/display_server/mir2/src/client/rpc/stream_socket_transport.cpp:40
#4 0x00007f71fff7362e in std::_Function_handler<void(const std::shared_ptr<mir::client::rpc::StreamTransport::Observer>&), mir::client::rpc::TransportObservers::on_data_available()::<lambda(auto:1)> >::_M_invoke(const std::_Any_data &, const std::shared_ptr<mir::client::rpc::StreamTransport::Observer> &)
    (__functor=..., __args#0=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<mir::client::rpc::MirProtobufRpcChannel*, mir::client::rpc::MirProtobufRpcChannel::MirProtobufRpcChannel(std::unique_ptr<mir::client...

Fix committed into lp:mir at revision None, scheduled for release in mir, milestone 0.19.0

mir (0.19.0+16.04.20160128-0ubuntu1) xenial; urgency=medium