Merge pull request #427 from kaymes/connection-lost-crash

Gracefully handle lost network connections
This commit is contained in:
Sasha Hilton 2020-01-30 02:04:37 +01:00 committed by GitHub
commit 83140bea88
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 153 additions and 45 deletions

View file

@ -333,7 +333,11 @@ impl Future for SpircTask {
progress = true; progress = true;
self.handle_frame(frame); self.handle_frame(frame);
} }
Async::Ready(None) => panic!("subscription terminated"), Async::Ready(None) => {
error!("subscription terminated");
self.shutdown = true;
self.commands.close();
}
Async::NotReady => (), Async::NotReady => (),
} }

View file

@ -14,6 +14,7 @@ component! {
download_rate_estimate: usize = 0, download_rate_estimate: usize = 0,
download_measurement_start: Option<Instant> = None, download_measurement_start: Option<Instant> = None,
download_measurement_bytes: usize = 0, download_measurement_bytes: usize = 0,
invalid: bool = false,
} }
} }
@ -46,7 +47,9 @@ impl ChannelManager {
let seq = self.lock(|inner| { let seq = self.lock(|inner| {
let seq = inner.sequence.get(); let seq = inner.sequence.get();
if !inner.invalid {
inner.channels.insert(seq, tx); inner.channels.insert(seq, tx);
}
seq seq
}); });
@ -87,12 +90,21 @@ impl ChannelManager {
pub fn get_download_rate_estimate(&self) -> usize { pub fn get_download_rate_estimate(&self) -> usize {
return self.lock(|inner| inner.download_rate_estimate); return self.lock(|inner| inner.download_rate_estimate);
} }
pub(crate) fn shutdown(&self) {
self.lock(|inner| {
inner.invalid = true;
// destroy the sending halves of the channels to signal everyone who is waiting for something.
inner.channels.clear();
});
}
} }
impl Channel { impl Channel {
fn recv_packet(&mut self) -> Poll<Bytes, ChannelError> { fn recv_packet(&mut self) -> Poll<Bytes, ChannelError> {
let (cmd, packet) = match self.receiver.poll() { let (cmd, packet) = match self.receiver.poll() {
Ok(Async::Ready(t)) => t.expect("channel closed"), Ok(Async::Ready(Some(t))) => t,
Ok(Async::Ready(None)) => return Err(ChannelError), // The channel has been closed.
Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::NotReady) => return Ok(Async::NotReady),
Err(()) => unreachable!(), Err(()) => unreachable!(),
}; };

View file

@ -28,9 +28,27 @@ pub fn connect(
let (addr, connect_url) = match *proxy { let (addr, connect_url) = match *proxy {
Some(ref url) => { Some(ref url) => {
info!("Using proxy \"{}\"", url); info!("Using proxy \"{}\"", url);
(url.to_socket_addrs().unwrap().next().unwrap(), Some(addr)) match url.to_socket_addrs().and_then(|mut iter| {
iter.next().ok_or(io::Error::new(
io::ErrorKind::NotFound,
"Can't resolve proxy server address",
))
}) {
Ok(socket_addr) => (socket_addr, Some(addr)),
Err(error) => return Box::new(futures::future::err(error)),
}
}
None => {
match addr.to_socket_addrs().and_then(|mut iter| {
iter.next().ok_or(io::Error::new(
io::ErrorKind::NotFound,
"Can't resolve server address",
))
}) {
Ok(socket_addr) => (socket_addr, None),
Err(error) => return Box::new(futures::future::err(error)),
}
} }
None => (addr.to_socket_addrs().unwrap().next().unwrap(), None),
}; };
let socket = TcpStream::connect(&addr, handle); let socket = TcpStream::connect(&addr, handle);

View file

@ -20,6 +20,7 @@ component! {
sequence: SeqGenerator<u64> = SeqGenerator::new(0), sequence: SeqGenerator<u64> = SeqGenerator::new(0),
pending: HashMap<Vec<u8>, MercuryPending> = HashMap::new(), pending: HashMap<Vec<u8>, MercuryPending> = HashMap::new(),
subscriptions: Vec<(String, mpsc::UnboundedSender<MercuryResponse>)> = Vec::new(), subscriptions: Vec<(String, mpsc::UnboundedSender<MercuryResponse>)> = Vec::new(),
invalid: bool = false,
} }
} }
@ -61,7 +62,11 @@ impl MercuryManager {
}; };
let seq = self.next_seq(); let seq = self.next_seq();
self.lock(|inner| inner.pending.insert(seq.clone(), pending)); self.lock(|inner| {
if !inner.invalid {
inner.pending.insert(seq.clone(), pending);
}
});
let cmd = req.method.command(); let cmd = req.method.command();
let data = req.encode(&seq); let data = req.encode(&seq);
@ -110,6 +115,7 @@ impl MercuryManager {
let (tx, rx) = mpsc::unbounded(); let (tx, rx) = mpsc::unbounded();
manager.lock(move |inner| { manager.lock(move |inner| {
if !inner.invalid {
debug!("subscribed uri={} count={}", uri, response.payload.len()); debug!("subscribed uri={} count={}", uri, response.payload.len());
if response.payload.len() > 0 { if response.payload.len() > 0 {
// Old subscription protocol, watch the provided list of URIs // Old subscription protocol, watch the provided list of URIs
@ -126,6 +132,7 @@ impl MercuryManager {
// New subscription protocol, watch the requested URI // New subscription protocol, watch the requested URI
inner.subscriptions.push((uri, tx)); inner.subscriptions.push((uri, tx));
} }
}
}); });
rx rx
@ -223,4 +230,13 @@ impl MercuryManager {
} }
} }
} }
pub(crate) fn shutdown(&self) {
self.lock(|inner| {
inner.invalid = true;
// destroy the sending halves of the channels to signal everyone who is waiting for something.
inner.pending.clear();
inner.subscriptions.clear();
});
}
} }

View file

@ -243,6 +243,8 @@ impl Session {
pub fn shutdown(&self) { pub fn shutdown(&self) {
debug!("Invalidating session[{}]", self.0.session_id); debug!("Invalidating session[{}]", self.0.session_id);
self.0.data.write().unwrap().invalid = true; self.0.data.write().unwrap().invalid = true;
self.mercury().shutdown();
self.channel().shutdown();
} }
pub fn is_invalid(&self) -> bool { pub fn is_invalid(&self) -> bool {
@ -289,14 +291,18 @@ where
loop { loop {
let (cmd, data) = match self.0.poll() { let (cmd, data) = match self.0.poll() {
Ok(Async::Ready(t)) => t, Ok(Async::Ready(Some(t))) => t,
Ok(Async::Ready(None)) => {
warn!("Connection to server closed.");
session.shutdown();
return Ok(Async::Ready(()));
}
Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::NotReady) => return Ok(Async::NotReady),
Err(e) => { Err(e) => {
session.shutdown(); session.shutdown();
return Err(From::from(e)); return Err(From::from(e));
} }
} };
.expect("connection closed");
session.dispatch(cmd, data); session.dispatch(cmd, data);
} }

View file

@ -636,9 +636,14 @@ impl PlayerInternal {
spotify_id: SpotifyId, spotify_id: SpotifyId,
position: i64, position: i64,
) -> Option<(Decoder, f32, StreamLoaderController, usize)> { ) -> Option<(Decoder, f32, StreamLoaderController, usize)> {
let audio = AudioItem::get_audio_item(&self.session, spotify_id) let audio = match AudioItem::get_audio_item(&self.session, spotify_id).wait() {
.wait() Ok(audio) => audio,
.unwrap(); Err(_) => {
error!("Unable to load audio item.");
return None;
}
};
info!("Loading <{}> with Spotify URI <{}>", audio.name, audio.uri); info!("Loading <{}> with Spotify URI <{}>", audio.name, audio.uri);
let audio = match self.find_available_alternative(&audio) { let audio = match self.find_available_alternative(&audio) {
@ -690,7 +695,13 @@ impl PlayerInternal {
play_from_beginning, play_from_beginning,
); );
let encrypted_file = encrypted_file.wait().unwrap(); let encrypted_file = match encrypted_file.wait() {
Ok(encrypted_file) => encrypted_file,
Err(_) => {
error!("Unable to load encrypted file.");
return None;
}
};
let mut stream_loader_controller = encrypted_file.get_stream_loader_controller(); let mut stream_loader_controller = encrypted_file.get_stream_loader_controller();
@ -702,7 +713,14 @@ impl PlayerInternal {
stream_loader_controller.set_random_access_mode(); stream_loader_controller.set_random_access_mode();
} }
let key = key.wait().unwrap(); let key = match key.wait() {
Ok(key) => key,
Err(_) => {
error!("Unable to load decryption key");
return None;
}
};
let mut decrypted_file = AudioDecrypt::new(key, encrypted_file); let mut decrypted_file = AudioDecrypt::new(key, encrypted_file);
let normalisation_factor = match NormalisationData::parse_from_file(&mut decrypted_file) { let normalisation_factor = match NormalisationData::parse_from_file(&mut decrypted_file) {

View file

@ -8,6 +8,7 @@ use std::mem;
use std::path::PathBuf; use std::path::PathBuf;
use std::process::exit; use std::process::exit;
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant;
use tokio_core::reactor::{Core, Handle}; use tokio_core::reactor::{Core, Handle};
use tokio_io::IoStream; use tokio_io::IoStream;
use url::Url; use url::Url;
@ -375,6 +376,8 @@ struct Main {
connect: Box<dyn Future<Item = Session, Error = io::Error>>, connect: Box<dyn Future<Item = Session, Error = io::Error>>,
shutdown: bool, shutdown: bool,
last_credentials: Option<Credentials>,
auto_connect_times: Vec<Instant>,
player_event_channel: Option<UnboundedReceiver<PlayerEvent>>, player_event_channel: Option<UnboundedReceiver<PlayerEvent>>,
player_event_program: Option<String>, player_event_program: Option<String>,
@ -398,6 +401,8 @@ impl Main {
spirc: None, spirc: None,
spirc_task: None, spirc_task: None,
shutdown: false, shutdown: false,
last_credentials: None,
auto_connect_times: Vec::new(),
signal: Box::new(tokio_signal::ctrl_c().flatten_stream()), signal: Box::new(tokio_signal::ctrl_c().flatten_stream()),
player_event_channel: None, player_event_channel: None,
@ -420,6 +425,7 @@ impl Main {
} }
fn credentials(&mut self, credentials: Credentials) { fn credentials(&mut self, credentials: Credentials) {
self.last_credentials = Some(credentials.clone());
let config = self.session_config.clone(); let config = self.session_config.clone();
let handle = self.handle.clone(); let handle = self.handle.clone();
@ -448,12 +454,14 @@ impl Future for Main {
if let Some(ref spirc) = self.spirc { if let Some(ref spirc) = self.spirc {
spirc.shutdown(); spirc.shutdown();
} }
self.auto_connect_times.clear();
self.credentials(creds); self.credentials(creds);
progress = true; progress = true;
} }
if let Async::Ready(session) = self.connect.poll().unwrap() { match self.connect.poll() {
Ok(Async::Ready(session)) => {
self.connect = Box::new(futures::future::empty()); self.connect = Box::new(futures::future::empty());
let mixer_config = self.mixer_config.clone(); let mixer_config = self.mixer_config.clone();
let mixer = (self.mixer)(Some(mixer_config)); let mixer = (self.mixer)(Some(mixer_config));
@ -475,6 +483,12 @@ impl Future for Main {
progress = true; progress = true;
} }
Ok(Async::NotReady) => (),
Err(error) => {
error!("Could not connect to server: {}", error);
self.connect = Box::new(futures::future::empty());
}
}
if let Async::Ready(Some(())) = self.signal.poll().unwrap() { if let Async::Ready(Some(())) = self.signal.poll().unwrap() {
trace!("Ctrl-C received"); trace!("Ctrl-C received");
@ -492,12 +506,32 @@ impl Future for Main {
progress = true; progress = true;
} }
let mut drop_spirc_and_try_to_reconnect = false;
if let Some(ref mut spirc_task) = self.spirc_task { if let Some(ref mut spirc_task) = self.spirc_task {
if let Async::Ready(()) = spirc_task.poll().unwrap() { if let Async::Ready(()) = spirc_task.poll().unwrap() {
if self.shutdown { if self.shutdown {
return Ok(Async::Ready(())); return Ok(Async::Ready(()));
} else { } else {
panic!("Spirc shut down unexpectedly"); warn!("Spirc shut down unexpectedly");
drop_spirc_and_try_to_reconnect = true;
}
progress = true;
}
}
if drop_spirc_and_try_to_reconnect {
self.spirc_task = None;
while (!self.auto_connect_times.is_empty())
&& ((Instant::now() - self.auto_connect_times[0]).as_secs() > 600)
{
let _ = self.auto_connect_times.remove(0);
}
if let Some(credentials) = self.last_credentials.clone() {
if self.auto_connect_times.len() >= 5 {
warn!("Spirc shut down too often. Not reconnecting automatically.");
} else {
self.auto_connect_times.push(Instant::now());
self.credentials(credentials);
} }
} }
} }