diff --git a/core/src/session.rs b/core/src/session.rs
index 5f08b134..58397c8d 100755
--- a/core/src/session.rs
+++ b/core/src/session.rs
@@ -252,8 +252,20 @@ impl Session {
             if session.is_invalid() {
                 break;
             }
-            let last_ping = session.0.data.read().last_ping.unwrap_or_else(Instant::now);
-            if last_ping.elapsed() >= SESSION_TIMEOUT {
+
+            // It would be so much easier to use elapsed but elapsed could
+            // potentially panic is rare cases.
+            // See:
+            // https://doc.rust-lang.org/std/time/struct.Instant.html#monotonicity
+            let now = Instant::now();
+
+            let last_ping = session.0.data.read().last_ping.unwrap_or(now);
+
+            let since_last_ping = now
+                .checked_duration_since(last_ping)
+                .unwrap_or(SESSION_TIMEOUT);
+
+            if since_last_ping >= SESSION_TIMEOUT {
                 session.shutdown();
                 // TODO: Optionally reconnect (with cached/last credentials?)
                 return Err(io::Error::new(
diff --git a/core/src/spclient.rs b/core/src/spclient.rs
index d6c5ffb1..88130317 100644
--- a/core/src/spclient.rs
+++ b/core/src/spclient.rs
@@ -114,8 +114,9 @@ impl SpClient {
         dst: &mut [u8],
     ) -> Result<(), Error> {
         // after a certain number of seconds, the challenge expires
-        const TIMEOUT: u64 = 5; // seconds
-        let now = Instant::now();
+        const TIMEOUT: Duration = Duration::from_secs(5);
+
+        let then = Instant::now();
 
         let md = Sha1::digest(ctx);
 
@@ -123,9 +124,18 @@ impl SpClient {
         let target: i64 = BigEndian::read_i64(&md[12..20]);
 
         let suffix = loop {
-            if now.elapsed().as_secs() >= TIMEOUT {
+            // It would be so much easier to use elapsed but elapsed could
+            // potentially panic is rare cases.
+            // See:
+            // https://doc.rust-lang.org/std/time/struct.Instant.html#monotonicity
+            if Instant::now()
+                .checked_duration_since(then)
+                .unwrap_or(TIMEOUT)
+                >= TIMEOUT
+            {
                 return Err(Error::deadline_exceeded(format!(
-                    "{TIMEOUT} seconds expired"
+                    "{} seconds expired",
+                    TIMEOUT.as_secs(),
                 )));
             }
 
diff --git a/src/main.rs b/src/main.rs
index 5590379d..3e06a0c0 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1836,7 +1836,16 @@ async fn main() {
                 warn!("Spirc shut down unexpectedly");
 
                 let mut reconnect_exceeds_rate_limit = || {
-                    auto_connect_times.retain(|&t| t.elapsed() < RECONNECT_RATE_LIMIT_WINDOW);
+                    // It would be so much easier to use elapsed but elapsed could
+                    // potentially panic is rare cases.
+                    // See:
+                    // https://doc.rust-lang.org/std/time/struct.Instant.html#monotonicity
+                    let now = Instant::now();
+
+                    auto_connect_times.retain(|&t| {
+                        now.checked_duration_since(t).unwrap_or(RECONNECT_RATE_LIMIT_WINDOW) < RECONNECT_RATE_LIMIT_WINDOW
+                    });
+
                     auto_connect_times.len() > RECONNECT_RATE_LIMIT
                 };