Skip to main content

autoconnect_ws_sm/identified/
mod.rs

1use std::{collections::HashMap, fmt, mem, sync::Arc};
2
3use actix_web::rt;
4use cadence::Timed;
5use futures::channel::mpsc;
6use once_cell::sync::Lazy;
7use tokio::sync::Semaphore;
8use uuid::Uuid;
9
10/// Limit concurrent disconnect-cleanup tasks to prevent resource exhaustion
11/// during disconnect storms (e.g., pod scaling events).
12static DISCONNECT_SEMAPHORE: Lazy<Semaphore> = Lazy::new(|| Semaphore::new(64));
13
14use autoconnect_common::{
15    broadcast::{Broadcast, BroadcastSubs},
16    protocol::{ServerMessage, ServerNotification},
17};
18
19use autoconnect_settings::{AppState, Settings};
20use autopush_common::{
21    db::User,
22    delivery_metrics::{DeliveryPath, Outcome, emit_outcomes},
23    metric_name::MetricName,
24    metrics::StatsdClientExt,
25    notification::Notification,
26    util::{ms_since_epoch, user_agent::UserAgentInfo},
27};
28
29use crate::error::{SMError, SMErrorKind};
30
31mod on_client_msg;
32mod on_server_notif;
33
34/// A WebPush Client that's successfully identified itself to the server via a
35/// Hello message.
36///
37/// The `webpush_ws` handler feeds input from both the WebSocket connection
38/// (`ClientMessage`) and the `ClientRegistry` (`ServerNotification`)
39/// triggered by autoendpoint to this type's `on_client_msg` and
40/// `on_server_notif` methods whose impls reside in their own modules.
41///
42/// Note the `check_storage` method (in the `on_server_notif` module) is
43/// triggered by both a `ServerNotification` and also the `new` constructor
44pub struct WebPushClient {
45    /// Push User Agent identifier. Each Push client recieves a unique UAID
46    pub uaid: Uuid,
47    /// Unique, local (to each autoconnect instance) identifier
48    pub uid: Uuid,
49    /// The User Agent information block derived from the User-Agent header
50    pub ua_info: UserAgentInfo,
51
52    /// Broadcast Subscriptions this Client is subscribed to
53    broadcast_subs: BroadcastSubs,
54
55    /// Set of session specific flags
56    flags: ClientFlags,
57    /// Notification Ack(knowledgement) related state
58    ack_state: AckState,
59    /// Count of messages sent from storage (for enforcing
60    /// `settings.msg_limit`). Resets to 0 when storage is emptied
61    sent_from_storage: u32,
62    /// Exists for new User records: these are not written to the db during
63    /// Hello, instead they're lazily added to the db on their first Register
64    /// message
65    deferred_add_user: Option<User>,
66
67    /// WebPush Session Statistics
68    stats: SessionStatistics,
69
70    /// Timestamp of when the UA connected (used by database lookup, thus u64)
71    connected_at: u64,
72    /// Timestamp of the last WebPush Ping message
73    last_ping: u64,
74    /// The last notification timestamp.
75    // TODO: RENAME THIS TO `last_notification_timestamp`
76    current_timestamp: Option<u64>,
77
78    app_state: Arc<AppState>,
79}
80
81impl fmt::Debug for WebPushClient {
82    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
83        fmt.debug_struct("WebPushClient")
84            .field("uaid", &self.uaid)
85            .field("uid", &self.uid)
86            .field("ua_info", &self.ua_info)
87            .field("broadcast_subs", &self.broadcast_subs)
88            .field("flags", &self.flags)
89            .field("ack_state", &self.ack_state)
90            .field("sent_from_storage", &self.sent_from_storage)
91            .field("deferred_add_user", &self.deferred_add_user)
92            .field("stats", &self.stats)
93            .field("connected_at", &self.connected_at)
94            .field("last_ping", &self.last_ping)
95            .finish()
96    }
97}
98
99impl WebPushClient {
100    #[allow(clippy::too_many_arguments)]
101    pub async fn new(
102        uaid: Uuid,
103        ua: String,
104        broadcast_subs: BroadcastSubs,
105        flags: ClientFlags,
106        connected_at: u64,
107        current_timestamp: Option<u64>,
108        deferred_add_user: Option<User>,
109        app_state: Arc<AppState>,
110    ) -> Result<(Self, Vec<ServerMessage>), SMError> {
111        trace!("👁‍🗨WebPushClient::new");
112        let stats = SessionStatistics {
113            existing_uaid: deferred_add_user.is_none(),
114            ..Default::default()
115        };
116        let mut client = WebPushClient {
117            uaid,
118            uid: Uuid::new_v4(),
119            ua_info: UserAgentInfo::from(ua.as_str()),
120            broadcast_subs,
121            flags,
122            ack_state: Default::default(),
123            sent_from_storage: Default::default(),
124            connected_at,
125            current_timestamp,
126            deferred_add_user,
127            last_ping: Default::default(),
128            stats,
129            app_state,
130        };
131
132        let smsgs = if client.flags.check_storage {
133            let smsgs = client.check_storage().await?;
134            debug!(
135                "WebPushClient::new: check_storage smsgs.len(): {}",
136                smsgs.len()
137            );
138            smsgs
139        } else {
140            vec![]
141        };
142        Ok((client, smsgs))
143    }
144
145    /// Return a reference to `AppState`'s `Settings`
146    pub fn app_settings(&self) -> &Settings {
147        &self.app_state.settings
148    }
149
150    #[cfg(feature = "reliable_report")]
151    pub fn app_reliability(&self) -> &autopush_common::reliability::PushReliability {
152        &self.app_state.reliability
153    }
154
155    /// Connect this `WebPushClient` to the `ClientRegistry`
156    ///
157    /// Returning a `Stream` of `ServerNotification`s from the `ClientRegistry`
158    pub fn registry_connect(&self) -> mpsc::Receiver<ServerNotification> {
159        self.app_state.clients.connect(self.uaid, self.uid)
160    }
161
162    /// Disconnect this `WebPushClient` from the `ClientRegistry`
163    pub fn registry_disconnect(&self) {
164        // Ignore disconnect (Client wasn't connected) Errors
165        let _ = self.app_state.clients.disconnect(&self.uaid, &self.uid);
166    }
167
168    /// Return the difference between the Client's Broadcast Subscriptions and
169    /// the this server's Broadcasts
170    pub async fn broadcast_delta(&mut self) -> Option<Vec<Broadcast>> {
171        self.app_state
172            .broadcaster
173            .read()
174            .await
175            .change_count_delta(&mut self.broadcast_subs)
176    }
177
178    /// Cleanup after the session has ended
179    pub fn shutdown(&mut self, reason: Option<String>) {
180        trace!("👁‍🗨WebPushClient::shutdown");
181        self.save_and_notify_unacked_direct_notifs();
182
183        let ua_info = &self.ua_info;
184        let stats = &self.stats;
185        let elapsed_sec = (ms_since_epoch() - self.connected_at) / 1_000;
186        self.app_state
187            .metrics
188            .time_with_tags("ua.connection.lifespan", elapsed_sec)
189            .with_tag("ua_os_family", &ua_info.metrics_os)
190            .with_tag("ua_browser_family", &ua_info.metrics_browser)
191            .send();
192
193        // Log out the final stats message
194        info!("Session";
195            "uaid_hash" => self.uaid.as_simple().to_string(),
196            "uaid_reset" => self.flags.old_record_version,
197            "existing_uaid" => stats.existing_uaid,
198            "connection_type" => "webpush",
199            "ua_name" => &ua_info.browser_name,
200            "ua_os_family" => &ua_info.metrics_os,
201            "ua_os_ver" => &ua_info.os_version,
202            "ua_browser_family" => &ua_info.metrics_browser,
203            "ua_browser_ver" => &ua_info.browser_version,
204            "ua_category" => &ua_info.category,
205            "connection_time" => elapsed_sec,
206            "direct_acked" => stats.direct_acked,
207            "direct_storage" => stats.direct_storage,
208            "stored_retrieved" => stats.stored_retrieved,
209            "stored_acked" => stats.stored_acked,
210            "nacks" => stats.nacks,
211            "registers" => stats.registers,
212            "unregisters" => stats.unregisters,
213            "disconnect_reason" => reason.unwrap_or_else(|| "".to_owned()),
214        );
215    }
216
217    /// Save any Direct unAck'd messages to the db (on shutdown)
218    ///
219    /// Direct messages are solely stored in memory until Ack'd by the Client,
220    /// so on shutdown, any not Ack'd are stored in the db to not be lost
221    fn save_and_notify_unacked_direct_notifs(&mut self) {
222        // TTL=0 notifications are delivered immediately or not at all
223        let ttl0_notifs = mem::take(&mut self.ack_state.unacked_ttl0_notifs);
224        self.expire_ttl0_notifs(ttl0_notifs.into_values().collect());
225        let notif_map = mem::take(&mut self.ack_state.unacked_direct_notifs);
226        trace!(
227            "👁‍🗨WebPushClient::save_and_notify_unacked_direct_notifs len: {}",
228            notif_map.len()
229        );
230        if notif_map.is_empty() {
231            return;
232        }
233
234        self.stats.direct_storage += notif_map.len() as i32;
235        // TODO: clarify this comment re the Python version
236        // Ensure we don't store these as legacy by setting a 0 as the
237        // sortkey_timestamp. This ensures the Python side doesn't mark it as
238        // legacy during conversion and still get the correct default us_time
239        // when saving
240        let mut notifs: Vec<Notification> = notif_map.into_values().collect();
241        for notif in &mut notifs {
242            notif.sortkey_timestamp = Some(0);
243        }
244
245        let app_state = Arc::clone(&self.app_state);
246        let uaid = self.uaid;
247        let connected_at = self.connected_at;
248        rt::spawn(async move {
249            let _permit = match DISCONNECT_SEMAPHORE.acquire().await {
250                Ok(permit) => permit,
251                Err(_) => {
252                    app_state
253                        .metrics
254                        .incr(MetricName::ErrorDisconnectSemaphoreFull)
255                        .ok();
256                    warn!("Disconnect semaphore full, skipping save of unacked direct notifs");
257                    return Ok(());
258                }
259            };
260            #[cfg(not(feature = "reliable_report"))]
261            app_state.db.save_messages(&uaid, notifs).await?;
262            #[cfg(feature = "reliable_report")]
263            {
264                app_state.db.save_messages(&uaid, notifs.clone()).await?;
265                for mut notif in notifs {
266                    notif
267                        .record_reliability(
268                            &app_state.reliability,
269                            autopush_common::reliability::ReliabilityState::Stored,
270                        )
271                        .await;
272                }
273            }
274            debug!("Finished saving unacked direct notifs, checking for reconnect");
275            let Some(user) = app_state.db.get_user(&uaid).await? else {
276                return Err(SMErrorKind::Internal(format!(
277                    "User not found for unacked direct notifs: {uaid}"
278                )));
279            };
280            if connected_at == user.connected_at {
281                return Ok(());
282            }
283            if let Some(node_id) = user.node_id {
284                app_state
285                    .http
286                    .put(format!("{}/notif/{}", node_id, uaid.as_simple()))
287                    .send()
288                    .await?
289                    .error_for_status()?;
290            }
291            Ok(())
292        });
293    }
294
295    /// Record TTL=0 notifications that were dropped without an ACK as expired
296    fn expire_ttl0_notifs(&self, notifs: Vec<Notification>) {
297        if notifs.is_empty() {
298            return;
299        }
300        emit_outcomes(
301            &self.app_state.metrics,
302            DeliveryPath::Ws,
303            Outcome::Expired,
304            notifs.len() as i64,
305        );
306        #[cfg(feature = "reliable_report")]
307        {
308            let app_state = Arc::clone(&self.app_state);
309            rt::spawn(async move {
310                for mut notif in notifs {
311                    notif
312                        .record_reliability(
313                            &app_state.reliability,
314                            autopush_common::reliability::ReliabilityState::Expired,
315                        )
316                        .await;
317                }
318            });
319        }
320    }
321
322    /// Add User information and tags for this Client to a Sentry Event
323    pub fn add_sentry_info(self, event: &mut sentry::protocol::Event) {
324        event.user = Some(sentry::User {
325            id: Some(self.uaid.as_simple().to_string()),
326            ..Default::default()
327        });
328        let ua_info = self.ua_info;
329        event
330            .tags
331            .insert("ua_name".to_owned(), ua_info.browser_name);
332        event
333            .tags
334            .insert("ua_os_family".to_owned(), ua_info.metrics_os);
335        event
336            .tags
337            .insert("ua_os_ver".to_owned(), ua_info.os_version);
338        event
339            .tags
340            .insert("ua_browser_family".to_owned(), ua_info.metrics_browser);
341        event
342            .tags
343            .insert("ua_browser_ver".to_owned(), ua_info.browser_version);
344    }
345}
346
347#[derive(Debug)]
348pub struct ClientFlags {
349    /// Whether check_storage queries for topic (not "timestamped") messages
350    pub include_topic: bool,
351    /// Flags the need to increment the last read for timestamp for timestamped messages
352    pub increment_storage: bool,
353    /// Whether this client needs to check storage for messages
354    pub check_storage: bool,
355    /// Flags the need to drop the user record
356    pub old_record_version: bool,
357    /// First time a user has connected "today"
358    pub emit_channel_metrics: bool,
359}
360
361impl Default for ClientFlags {
362    fn default() -> Self {
363        Self {
364            include_topic: true,
365            increment_storage: false,
366            check_storage: false,
367            old_record_version: false,
368            emit_channel_metrics: false,
369        }
370    }
371}
372
373/// WebPush Session Statistics
374///
375/// Tracks statistics about the session that are logged when the session's
376/// closed
377#[derive(Debug, Default)]
378pub struct SessionStatistics {
379    /// Number of acknowledged messages that were sent directly (not via storage)
380    direct_acked: i32,
381    /// Number of messages sent to storage
382    direct_storage: i32,
383    /// Number of messages taken from storage
384    stored_retrieved: i32,
385    /// Number of message pulled from storage and acknowledged
386    stored_acked: i32,
387    /// Number of messages total that are not acknowledged.
388    nacks: i32,
389    /// Number of unregister requests
390    unregisters: i32,
391    /// Number of register requests
392    registers: i32,
393    /// Whether this uaid was previously registered
394    existing_uaid: bool,
395}
396
397/// Key for looking up notifications in the ACK tracking maps.
398/// The `version` field is the `Notification.message_id`, a fernet-encrypted
399/// composite of uaid, channel_id, and topic|timestamp. It is globally unique
400/// (fernet adds random padding), so `version` alone suffices as the key.
401type AckKey = String;
402
403/// A Notification the Client has Ack'd, and when.
404#[derive(Debug)]
405struct AckedNotification {
406    notif: Notification,
407    ack_ms: u64,
408}
409
410/// Record of Notifications sent to the Client.
411#[derive(Debug, Default)]
412struct AckState {
413    /// Map of unAck'd directly sent (never stored) notifications
414    unacked_direct_notifs: HashMap<AckKey, Notification>,
415    /// Map of unAck'd directly sent TTL=0 notifications, kept only to record
416    /// their delivery on ACK. Unlike `unacked_direct_notifs`, these are never
417    /// stored for redelivery.
418    unacked_ttl0_notifs: HashMap<AckKey, Notification>,
419    /// Map of unAck'd sent notifications from storage
420    unacked_stored_notifs: HashMap<AckKey, Notification>,
421    /// Ack'd timestamp notifications from storage, with when they were Ack'd.
422    /// The whole batch must be Ack'd before messages are removed from storage
423    /// via `increment_storage`; if the session ends first, they will be sent
424    /// again on reconnect, so their delivery is recorded in `increment_storage`.
425    acked_stored_timestamp_notifs: Vec<AckedNotification>,
426    /// Expired timestamp notifications skipped while reading storage. Like
427    /// `acked_stored_timestamp_notifs`, they're counted in `increment_storage`.
428    expired_stored_timestamp_count: i64,
429    /// Either the `current_timestamp` value in storage (returned from
430    /// `fetch_messages`) or the last unAck'd timestamp Message's
431    /// `sortkey_timestamp` (returned from `fetch_timestamp_messages`).
432    ///
433    /// This represents the "pointer" to the beginning (more specifically the
434    /// record preceeding the beginning used in a Greater Than query) of the
435    /// next batch of timestamp Messages.
436    ///
437    /// Thus this value is:
438    ///
439    /// a) initially None, then
440    ///
441    /// b) retrieved from `current_timestamp` in storage then passed as the
442    /// `timestamp` to `fetch_timestamp_messages`. When all of those timestamp
443    /// Messages are Ack'd, this value's then
444    ///
445    /// c) written back to `current_timestamp` in storage via
446    /// `increment_storage`
447    unacked_stored_highest: Option<u64>,
448}
449
450impl AckState {
451    /// Whether the Client has outstanding notifications sent to it that it has
452    /// yet to Ack
453    ///
454    /// TTL=0 notifications are not included because they are not stored
455    /// for redelivery.
456    fn unacked_notifs(&self) -> bool {
457        !self.unacked_stored_notifs.is_empty() || !self.unacked_direct_notifs.is_empty()
458    }
459}
460
461#[cfg(test)]
462mod tests {
463    use std::sync::Arc;
464
465    use cadence::{SpyMetricSink, StatsdClient};
466    use uuid::Uuid;
467
468    use autoconnect_common::{
469        protocol::{ClientAck, ClientMessage, ServerMessage, ServerNotification},
470        test_support::{DUMMY_CHID, DUMMY_UAID, UA},
471    };
472    use autoconnect_settings::AppState;
473    use autopush_common::{
474        db::{client::FetchMessageResponse, error::DbError, mock::MockDbClient},
475        notification::Notification,
476        util::{ms_since_epoch, sec_since_epoch},
477    };
478
479    use super::WebPushClient;
480
481    async fn wpclient(uaid: Uuid, app_state: AppState) -> (WebPushClient, Vec<ServerMessage>) {
482        WebPushClient::new(
483            uaid,
484            UA.to_owned(),
485            Default::default(),
486            Default::default(),
487            ms_since_epoch(),
488            None,
489            None,
490            Arc::new(app_state),
491        )
492        .await
493        .unwrap()
494    }
495
496    /// Generate a dummy timestamp `Notification`
497    fn new_timestamp_notif(channel_id: &Uuid, ttl: u64) -> Notification {
498        Notification {
499            channel_id: *channel_id,
500            ttl,
501            timestamp: sec_since_epoch(),
502            sortkey_timestamp: Some(ms_since_epoch()),
503            ..Default::default()
504        }
505    }
506
507    /// An `AppState` using `db` whose statsd metrics are captured by the
508    /// returned receiver
509    fn spy_app_state(db: MockDbClient) -> (AppState, impl Fn() -> Vec<String>) {
510        let (rx, sink) = SpyMetricSink::new();
511        let app_state = AppState {
512            db: db.into_boxed_arc(),
513            metrics: Arc::new(StatsdClient::from_sink("autoconnect", sink)),
514            ..Default::default()
515        };
516        let packets = move || {
517            rx.try_iter()
518                .map(|packet| String::from_utf8(packet).unwrap())
519                .collect()
520        };
521        (app_state, packets)
522    }
523
524    /// Sum of `notification.outcome` for `outcome` in the statsd `packets`
525    fn outcome_count(packets: &[String], outcome: &str) -> i64 {
526        let tag = format!("outcome:{outcome},");
527        packets
528            .iter()
529            .filter(|p| p.starts_with("autoconnect.notification.outcome:") && p.contains(&tag))
530            .map(|p| {
531                p["autoconnect.notification.outcome:".len()..]
532                    .split('|')
533                    .next()
534                    .unwrap()
535                    .parse::<i64>()
536                    .unwrap()
537            })
538            .sum()
539    }
540
541    /// A topic `Notification` (no `sortkey_timestamp`)
542    fn new_topic_notif(channel_id: &Uuid, ttl: u64) -> Notification {
543        Notification {
544            topic: Some("topic".to_owned()),
545            sortkey_timestamp: None,
546            ..new_timestamp_notif(channel_id, ttl)
547        }
548    }
549
550    fn ack(version: &str) -> ClientMessage {
551        ClientMessage::Ack {
552            updates: vec![ClientAck {
553                channel_id: DUMMY_CHID,
554                version: version.to_owned(),
555                code: Some(100),
556            }],
557        }
558    }
559
560    /// Mock reading one batch of timestamp messages (`v1`, `v2` and an
561    /// expired one) at `timestamp`, with no topic messages
562    fn expect_timestamp_batch(db: &mut MockDbClient, seq: &mut mockall::Sequence, timestamp: u64) {
563        db.expect_fetch_topic_messages()
564            .times(1)
565            .in_sequence(seq)
566            .return_once(|_, _| Ok(FetchMessageResponse::default()));
567        db.expect_fetch_timestamp_messages()
568            .times(1)
569            .in_sequence(seq)
570            .withf(|_, ts, _| ts.is_none())
571            .return_once(move |_, _, _| {
572                Ok(FetchMessageResponse {
573                    timestamp: Some(timestamp),
574                    messages: vec![
575                        Notification {
576                            version: "v1".to_owned(),
577                            ..new_timestamp_notif(&DUMMY_CHID, 60)
578                        },
579                        Notification {
580                            version: "v2".to_owned(),
581                            ..new_timestamp_notif(&DUMMY_CHID, 60)
582                        },
583                        Notification {
584                            version: "expired".to_owned(),
585                            ..new_timestamp_notif(&DUMMY_CHID, 0)
586                        },
587                    ],
588                })
589            });
590    }
591
592    /// A stored batch that's only partly Ack'd before the session ends is
593    /// read again on reconnect, so nothing in it is counted yet
594    #[actix_rt::test]
595    async fn partial_stored_ack_counts_nothing() {
596        let mut db = MockDbClient::new();
597        let mut seq = mockall::Sequence::new();
598        expect_timestamp_batch(&mut db, &mut seq, sec_since_epoch());
599        db.expect_increment_storage().never();
600        let (app_state, packets) = spy_app_state(db);
601        let delivery_metrics = app_state.delivery_metrics.clone();
602        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
603
604        let smsgs = client
605            .on_server_notif(ServerNotification::CheckStorage)
606            .await
607            .unwrap();
608        assert_eq!(smsgs.len(), 2);
609        client.on_client_msg(ack("v1")).await.unwrap();
610        client.shutdown(None);
611
612        let packets = packets();
613        assert_eq!(outcome_count(&packets, "delivered"), 0);
614        assert_eq!(outcome_count(&packets, "expired"), 0);
615        assert!(
616            !delivery_metrics
617                .encode()
618                .unwrap()
619                .contains(r#"route="stored""#)
620        );
621    }
622
623    /// Once a stored batch is fully Ack'd and storage advances, each message
624    /// is counted exactly once
625    #[actix_rt::test]
626    async fn full_stored_ack_counts_once() {
627        let mut db = MockDbClient::new();
628        let mut seq = mockall::Sequence::new();
629        let timestamp = sec_since_epoch();
630        expect_timestamp_batch(&mut db, &mut seq, timestamp);
631        db.expect_increment_storage()
632            .times(1)
633            .in_sequence(&mut seq)
634            .withf(move |_, ts| ts == &timestamp)
635            .return_once(|_, _| Ok(()));
636        db.expect_fetch_timestamp_messages()
637            .times(1)
638            .in_sequence(&mut seq)
639            .withf(move |_, ts, _| ts == &Some(timestamp))
640            .return_once(|_, _, _| Ok(FetchMessageResponse::default()));
641        let (app_state, packets) = spy_app_state(db);
642        let delivery_metrics = app_state.delivery_metrics.clone();
643        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
644
645        client
646            .on_server_notif(ServerNotification::CheckStorage)
647            .await
648            .unwrap();
649        client.on_client_msg(ack("v1")).await.unwrap();
650        // Nothing is counted until the whole batch is Ack'd
651        assert_eq!(outcome_count(&packets(), "delivered"), 0);
652        client.on_client_msg(ack("v2")).await.unwrap();
653
654        let packets = packets();
655        assert_eq!(outcome_count(&packets, "delivered"), 2);
656        assert_eq!(outcome_count(&packets, "expired"), 1);
657        let report = delivery_metrics.encode().unwrap();
658        assert!(report.contains(r#"autopush_delivery_seconds_count{path="ws",route="stored"} 2"#));
659        assert!(report.contains("autopush_stored_delivery_after_connect_seconds_count 2"));
660    }
661
662    /// An Ack'd topic message is only counted once it's removed from storage
663    #[actix_rt::test]
664    async fn acked_topic_counted_after_remove() {
665        let mut db = MockDbClient::new();
666        db.expect_fetch_topic_messages()
667            .times(1)
668            .return_once(|_, _| {
669                Ok(FetchMessageResponse {
670                    timestamp: None,
671                    messages: vec![Notification {
672                        version: "topic".to_owned(),
673                        ..new_topic_notif(&DUMMY_CHID, 60)
674                    }],
675                })
676            });
677        db.expect_remove_message()
678            .times(1)
679            .return_once(|_, _| Err(DbError::General("remove failed".to_owned())));
680        let (app_state, packets) = spy_app_state(db);
681        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
682
683        client
684            .on_server_notif(ServerNotification::CheckStorage)
685            .await
686            .unwrap();
687        client.on_client_msg(ack("topic")).await.unwrap_err();
688        assert_eq!(outcome_count(&packets(), "delivered"), 0);
689    }
690
691    /// An expired topic message is only counted once it's removed from storage
692    #[actix_rt::test]
693    async fn expired_topic_counted_after_remove() {
694        let mut db = MockDbClient::new();
695        db.expect_fetch_topic_messages()
696            .times(1)
697            .return_once(|_, _| {
698                Ok(FetchMessageResponse {
699                    timestamp: None,
700                    messages: vec![new_topic_notif(&DUMMY_CHID, 0)],
701                })
702            });
703        db.expect_remove_message()
704            .times(1)
705            .return_once(|_, _| Err(DbError::General("remove failed".to_owned())));
706        let (app_state, packets) = spy_app_state(db);
707        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
708
709        client
710            .on_server_notif(ServerNotification::CheckStorage)
711            .await
712            .unwrap_err();
713        assert_eq!(outcome_count(&packets(), "expired"), 0);
714    }
715
716    #[actix_rt::test]
717    async fn webpush_ping() {
718        let (mut client, _) = wpclient(DUMMY_UAID, Default::default()).await;
719        let pong = client.on_client_msg(ClientMessage::Ping).await.unwrap();
720        assert!(matches!(pong.as_slice(), [ServerMessage::Ping]));
721    }
722
723    #[actix_rt::test]
724    async fn direct_ack_records_delivery() {
725        let app_state = AppState::default();
726        let delivery_metrics = app_state.delivery_metrics.clone();
727        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
728        let notif = Notification {
729            version: "v1".to_owned(),
730            received_ms: Some(ms_since_epoch() - 2_000),
731            ..new_timestamp_notif(&DUMMY_CHID, 60)
732        };
733        let smsgs = client
734            .on_server_notif(ServerNotification::Notification(notif))
735            .await
736            .unwrap();
737        assert!(matches!(smsgs.as_slice(), [ServerMessage::Notification(_)]));
738
739        client
740            .on_client_msg(ClientMessage::Ack {
741                updates: vec![ClientAck {
742                    channel_id: DUMMY_CHID,
743                    version: "v1".to_owned(),
744                    code: Some(101),
745                }],
746            })
747            .await
748            .unwrap();
749
750        let report = delivery_metrics.encode().unwrap();
751        // Received 2s before the ACK
752        assert!(
753            report.contains(
754                r#"autopush_delivery_seconds_bucket{le="1.0",path="ws",route="direct"} 0"#
755            )
756        );
757        assert!(
758            report.contains(
759                r#"autopush_delivery_seconds_bucket{le="2.5",path="ws",route="direct"} 1"#
760            )
761        );
762        // Not a stored message
763        assert!(report.contains("autopush_stored_delivery_after_connect_seconds_count 0"));
764    }
765
766    #[actix_rt::test]
767    async fn direct_ttl0_ack_records_delivery_and_is_not_stored() {
768        let mut db = MockDbClient::new();
769        // TTL=0 notifications are never saved on shutdown, whether unacked or
770        // still queued
771        db.expect_save_messages().never();
772        let app_state = AppState {
773            db: db.into_boxed_arc(),
774            ..Default::default()
775        };
776        let delivery_metrics = app_state.delivery_metrics.clone();
777        let (mut client, _) = wpclient(DUMMY_UAID, app_state).await;
778        let notif = Notification {
779            version: "v1".to_owned(),
780            received_ms: Some(ms_since_epoch()),
781            ..new_timestamp_notif(&DUMMY_CHID, 0)
782        };
783        client
784            .on_server_notif(ServerNotification::Notification(notif))
785            .await
786            .unwrap();
787        // A second TTL=0 notification left unacked at shutdown
788        let notif = Notification {
789            version: "v2".to_owned(),
790            ..new_timestamp_notif(&DUMMY_CHID, 0)
791        };
792        client
793            .on_server_notif(ServerNotification::Notification(notif))
794            .await
795            .unwrap();
796
797        client
798            .on_client_msg(ClientMessage::Ack {
799                updates: vec![ClientAck {
800                    channel_id: DUMMY_CHID,
801                    version: "v1".to_owned(),
802                    code: Some(100),
803                }],
804            })
805            .await
806            .unwrap();
807        let report = delivery_metrics.encode().unwrap();
808        assert!(report.contains(r#"autopush_delivery_seconds_count{path="ws",route="direct"} 1"#));
809
810        // A TTL=0 notification still queued when the session ends
811        client.on_server_notif_shutdown(ServerNotification::Notification(Notification {
812            version: "v3".to_owned(),
813            ..new_timestamp_notif(&DUMMY_CHID, 0)
814        }));
815        client.shutdown(None);
816    }
817
818    /// An unacked TTL=0 notification must not stop storage processing
819    #[actix_rt::test]
820    async fn unacked_ttl0_does_not_block_storage() {
821        let mut db = MockDbClient::new();
822        let mut seq = mockall::Sequence::new();
823        let timestamp = sec_since_epoch();
824        let stored = Notification {
825            version: "stored".to_owned(),
826            ..new_timestamp_notif(&DUMMY_CHID, 60)
827        };
828        db.expect_fetch_topic_messages()
829            .times(1)
830            .in_sequence(&mut seq)
831            .return_once(|_, _| Ok(FetchMessageResponse::default()));
832        db.expect_fetch_timestamp_messages()
833            .times(1)
834            .in_sequence(&mut seq)
835            .return_once(move |_, _, _| {
836                Ok(FetchMessageResponse {
837                    timestamp: Some(timestamp),
838                    messages: vec![stored],
839                })
840            });
841        // Acking the stored message advances storage and reads the next batch
842        db.expect_increment_storage()
843            .times(1)
844            .in_sequence(&mut seq)
845            .withf(move |_, ts| ts == &timestamp)
846            .return_once(|_, _| Ok(()));
847        db.expect_fetch_timestamp_messages()
848            .times(1)
849            .in_sequence(&mut seq)
850            .withf(move |_, ts, _| ts == &Some(timestamp))
851            .return_once(|_, _, _| Ok(FetchMessageResponse::default()));
852        db.expect_save_messages().never();
853
854        let (mut client, _) = wpclient(
855            DUMMY_UAID,
856            AppState {
857                db: db.into_boxed_arc(),
858                ..Default::default()
859            },
860        )
861        .await;
862
863        // A TTL=0 notification the client never acks
864        client
865            .on_server_notif(ServerNotification::Notification(Notification {
866                version: "ttl0".to_owned(),
867                ..new_timestamp_notif(&DUMMY_CHID, 0)
868            }))
869            .await
870            .unwrap();
871        let smsgs = client
872            .on_server_notif(ServerNotification::CheckStorage)
873            .await
874            .unwrap();
875        assert!(matches!(smsgs.as_slice(), [ServerMessage::Notification(_)]));
876
877        client
878            .on_client_msg(ClientMessage::Ack {
879                updates: vec![ClientAck {
880                    channel_id: DUMMY_CHID,
881                    version: "stored".to_owned(),
882                    code: Some(100),
883                }],
884            })
885            .await
886            .unwrap();
887        client.shutdown(None);
888    }
889
890    #[actix_rt::test]
891    async fn expired_increments_storage() {
892        let mut db = MockDbClient::new();
893        let mut seq = mockall::Sequence::new();
894        let timestamp = sec_since_epoch();
895        // No topic messages
896        db.expect_fetch_topic_messages()
897            .times(1)
898            .in_sequence(&mut seq)
899            .return_once(move |_, _| {
900                Ok(FetchMessageResponse {
901                    timestamp: None,
902                    messages: vec![],
903                })
904            });
905        // Return expired notifs (default ttl of 0)
906        db.expect_fetch_timestamp_messages()
907            .times(1)
908            .in_sequence(&mut seq)
909            .withf(move |_, ts, _| ts.is_none())
910            .return_once(move |_, _, _| {
911                Ok(FetchMessageResponse {
912                    timestamp: Some(timestamp),
913                    messages: vec![
914                        new_timestamp_notif(&DUMMY_CHID, 0),
915                        new_timestamp_notif(&DUMMY_CHID, 0),
916                    ],
917                })
918            });
919        // EOF
920        db.expect_fetch_timestamp_messages()
921            .times(1)
922            .in_sequence(&mut seq)
923            .withf(move |_, ts, _| ts == &Some(timestamp))
924            .return_once(|_, _, _| {
925                Ok(FetchMessageResponse {
926                    timestamp: None,
927                    messages: vec![],
928                })
929            });
930        // Ensure increment_storage's called to advance the timestamp messages
931        // despite check_storage returning nothing (all filtered out as
932        // expired)
933        db.expect_increment_storage()
934            .times(1)
935            .in_sequence(&mut seq)
936            .withf(move |_, ts| ts == &timestamp)
937            .return_once(|_, _| Ok(()));
938
939        // No check_storage called here (via default ClientFlags)
940        let (mut client, _) = wpclient(
941            DUMMY_UAID,
942            AppState {
943                db: db.into_boxed_arc(),
944                ..Default::default()
945            },
946        )
947        .await;
948
949        let smsgs = client
950            .on_server_notif(ServerNotification::CheckStorage)
951            .await
952            .expect("CheckStorage failed");
953        assert!(smsgs.is_empty())
954    }
955}