Skip to main content

autoconnect_ws_sm/identified/
on_server_notif.rs

1use std::mem;
2
3use cadence::{Counted, Timed};
4
5use autoconnect_common::protocol::{ServerMessage, ServerNotification};
6use autopush_common::{
7    db::CheckStorageResponse,
8    delivery_metrics::{DeliveryPath, DeliveryRoute, Outcome, emit_outcome, emit_outcomes},
9    metric_name::MetricName,
10    metrics::StatsdClientExt,
11    notification::Notification,
12    util::sec_since_epoch,
13};
14
15use super::WebPushClient;
16use crate::error::{SMError, SMErrorKind};
17
18/// Where a Notification being sent to the client originated, used to tag send
19/// metrics and to gate storage-only measurements.
20#[derive(Clone, Copy, PartialEq, Eq)]
21enum SendSource {
22    /// Pushed live to a connected client (never entered storage).
23    Direct,
24    /// Retrieved from storage on (re)connect.
25    Stored,
26}
27
28impl SendSource {
29    /// The tag value emitted to metrics.
30    fn as_tag(&self) -> &'static str {
31        match self {
32            SendSource::Direct => "Direct",
33            SendSource::Stored => "Stored",
34        }
35    }
36}
37
38impl WebPushClient {
39    /// Handle a `ServerNotification` for this user
40    ///
41    /// `ServerNotification::Disconnect` is emitted by the same autoconnect
42    /// node receiving it when a User has logged into that same node twice to
43    /// "Ghost" (disconnect) the first user's session for its second session.
44    ///
45    /// Other variants are emitted by autoendpoint
46    pub async fn on_server_notif(
47        &mut self,
48        snotif: ServerNotification,
49    ) -> Result<Vec<ServerMessage>, SMError> {
50        match snotif {
51            ServerNotification::Notification(notif) => Ok(vec![self.notif(notif)?]),
52            ServerNotification::CheckStorage => self.check_storage().await,
53            ServerNotification::Disconnect => Err(SMErrorKind::Ghost.into()),
54        }
55    }
56
57    /// After disconnecting from the `ClientRegistry`, moves any queued Direct
58    /// Push Notifications to unacked_direct_notifs (to be stored in the db on
59    /// `shutdown`). TTL=0 notifications are dropped as expired, since they
60    /// can't be delivered later.
61    pub fn on_server_notif_shutdown(&mut self, snotif: ServerNotification) {
62        trace!("WebPushClient::on_server_notif_shutdown");
63        if let ServerNotification::Notification(notif) = snotif {
64            if notif.ttl == 0 {
65                self.expire_ttl0_notifs(vec![notif]);
66                return;
67            }
68            let key = notif.version.clone();
69            self.ack_state.unacked_direct_notifs.insert(key, notif);
70        }
71    }
72
73    /// Send a Direct Push Notification to this user
74    fn notif(&mut self, notif: Notification) -> Result<ServerMessage, SMError> {
75        trace!("WebPushClient::notif Sending a direct notif");
76        // The notification we return here is sent directly to the client.
77        // No reliability state is recorded.
78        let response = notif.clone();
79        // Consume the original notification by adding it to an unacked map so
80        // the client's ACK records its delivery. TTL=0 notifications go in a
81        // separate map: they're never stored, and don't hold up storage
82        // processing while unacked.
83        let key = notif.version.clone();
84        if notif.ttl != 0 {
85            self.ack_state.unacked_direct_notifs.insert(key, notif);
86        } else {
87            self.ack_state.unacked_ttl0_notifs.insert(key, notif);
88        }
89        self.emit_send_metrics(&response, SendSource::Direct);
90        Ok(ServerMessage::Notification(response))
91    }
92
93    /// Top level read of Push Notifications from storage
94    ///
95    /// Initializes the top level `check_storage` and `include_topic` flags and
96    /// runs `check_storage_loop`
97    pub(super) async fn check_storage(&mut self) -> Result<Vec<ServerMessage>, SMError> {
98        trace!("🗄️ WebPushClient::check_storage");
99        self.flags.check_storage = true;
100        self.flags.include_topic = true;
101        self.check_storage_loop().await
102    }
103
104    /// Loop the read of Push Notifications from storage
105    ///
106    /// Loops until any unexpired Push Notifications are read or there's no
107    /// more Notifications in storage
108    pub(super) async fn check_storage_loop(&mut self) -> Result<Vec<ServerMessage>, SMError> {
109        trace!("🗄️ WebPushClient::check_storage_loop");
110        while self.flags.check_storage {
111            let smsgs = self.check_storage_advance().await?;
112            if !smsgs.is_empty() {
113                self.check_msg_limit().await?;
114                return Ok(smsgs);
115            }
116        }
117        // No more notifications (check_storage = false). Despite returning no
118        // notifs we may have advanced through expired timestamp messages and
119        // need to increment_storage to mark them as deleted
120        if self.flags.increment_storage {
121            debug!("🗄️ WebPushClient::check_storage_loop increment_storage");
122            self.increment_storage().await?;
123        }
124        Ok(vec![])
125    }
126
127    /// Read a chunk (max count 10 returned) of Notifications from storage
128    ///
129    /// This filters out expired Notifications and may return an empty result
130    /// set when there's still pending Notifications to be read: so it should
131    /// be called in a loop to advance through all Notification records
132    async fn check_storage_advance(&mut self) -> Result<Vec<ServerMessage>, SMError> {
133        trace!("🗄️ WebPushClient::check_storage_advance");
134        let CheckStorageResponse {
135            include_topic,
136            mut messages,
137            timestamp,
138        } = self.do_check_storage().await?;
139
140        debug!(
141            "🗄️ WebPushClient::check_storage_advance \
142                 include_topic: {} -> {} \
143                 unacked_stored_highest: {:?} -> {:?}",
144            self.flags.include_topic,
145            include_topic,
146            self.ack_state.unacked_stored_highest,
147            timestamp
148        );
149        self.flags.include_topic = include_topic;
150        self.ack_state.unacked_stored_highest = timestamp;
151
152        if messages.is_empty() {
153            trace!("🗄️ WebPushClient::check_storage_advance finished");
154            self.flags.check_storage = false;
155            self.sent_from_storage = 0;
156            return Ok(vec![]);
157        }
158
159        // Filter out TTL expired messages
160        let now_sec = sec_since_epoch();
161        // Topic messages require immediate deletion from the db
162        let mut expired_messages = vec![];
163        // Expired timestamp messages are only skipped for good once
164        // increment_storage advances past them, so they're counted there
165        let mut expired_timestamp_count = 0;
166        // NOTE: Vec::extract_if (stabilizing soon) can negate the need for the
167        // inner msg.clone()
168        messages.retain(|msg| {
169            if !msg.expired(now_sec) {
170                return true;
171            }
172            if msg.sortkey_timestamp.is_none() {
173                expired_messages.push(msg.clone());
174            } else {
175                expired_timestamp_count += 1;
176            }
177            false
178        });
179        self.ack_state.expired_stored_timestamp_count += expired_timestamp_count;
180        // TODO: A batch remove_messages would be nicer
181        #[allow(unused_mut)]
182        for mut msg in expired_messages {
183            let chidmessageid = msg.chidmessageid();
184            trace!("🉑 removing expired topic chidmessageid: {chidmessageid}");
185            self.app_state
186                .db
187                .remove_message(&self.uaid, &chidmessageid)
188                .await?;
189            emit_outcome(&self.app_state.metrics, DeliveryPath::Ws, Outcome::Expired);
190            #[cfg(feature = "reliable_report")]
191            msg.record_reliability(
192                &self.app_state.reliability,
193                autopush_common::reliability::ReliabilityState::Expired,
194            )
195            .await;
196        }
197
198        self.flags.increment_storage = !include_topic && timestamp.is_some();
199
200        if messages.is_empty() {
201            trace!("🗄️ WebPushClient::check_storage_advance empty response (filtered expired)");
202            return Ok(vec![]);
203        }
204
205        for msg in messages.iter() {
206            let key = msg.version.clone();
207            self.ack_state
208                .unacked_stored_notifs
209                .insert(key, msg.clone());
210        }
211        let smsgs: Vec<_> = messages
212            .into_iter()
213            .inspect(|msg| {
214                trace!("🗄️ WebPushClient::check_storage_advance Sending stored");
215                self.emit_send_metrics(msg, SendSource::Stored)
216            })
217            .map(ServerMessage::Notification)
218            .collect();
219
220        let count = smsgs.len() as u32;
221        debug!(
222            "🗄️ WebPushClient::check_storage_advance: sent_from_storage: {}, +{}",
223            self.sent_from_storage, count
224        );
225        self.sent_from_storage += count;
226        Ok(smsgs)
227    }
228
229    #[cfg(feature = "reliable_report")]
230    /// Record and transition the state for trackable messages.
231    async fn record_state(
232        &self,
233        messages: &mut Vec<Notification>,
234        state: autopush_common::reliability::ReliabilityState,
235    ) {
236        // *Note* because `.map()` is sync
237        // we can't call the async func without additional hoops.
238        for message in messages {
239            message
240                .record_reliability(&self.app_state.reliability, state)
241                .await;
242        }
243    }
244
245    /// Read a chunk (max count 10 returned) of Notifications from storage
246    ///
247    /// This alternates between reading Topic Notifications and Timestamp
248    /// Notifications which are stored separately in storage.
249    ///
250    /// Topic Messages differ in that they replace pending Notifications with
251    /// new ones if they have matching Topic names. They are used when a sender
252    /// desires a scenario where multiple Messages sent to an offline device
253    /// result in the user only seeing the latest Message when the device comes
254    /// online.
255    async fn do_check_storage(&self) -> Result<CheckStorageResponse, SMError> {
256        // start at the latest unacked timestamp or the previous, latest timestamp.
257        let timestamp = self
258            .ack_state
259            .unacked_stored_highest
260            .or(self.current_timestamp);
261        trace!("🗄️ WebPushClient::do_check_storage {:?}", &timestamp);
262        // if we're to include topic messages, do those first.
263        // NOTE: Bigtable can't fetch `current_timestamp`, so we can't rely on
264        // `fetch_topic_messages()` returning a reasonable timestamp.
265        let topic_resp = if self.flags.include_topic {
266            trace!("🗄️ WebPushClient::do_check_storage: fetch_topic_messages");
267            // Get the most recent max 11 messages.
268            #[allow(unused_mut)]
269            let mut messages = self
270                .app_state
271                .db
272                .fetch_topic_messages(&self.uaid, 11)
273                .await?;
274            #[cfg(feature = "reliable_report")]
275            // Since we pulled these from storage, mark them as "retrieved"
276            self.record_state(
277                &mut messages.messages,
278                autopush_common::reliability::ReliabilityState::Retrieved,
279            )
280            .await;
281            messages
282        } else {
283            Default::default()
284        };
285        // if we have topic messages...
286        if !topic_resp.messages.is_empty() {
287            trace!(
288                "🗄️ WebPushClient::do_check_storage: Topic message returns: {:#?}",
289                topic_resp.messages
290            );
291            self.app_state
292                .metrics
293                .count_with_tags(
294                    "notification.message.retrieved",
295                    topic_resp.messages.len() as i64,
296                )
297                .with_tag("topic", "true")
298                .send();
299            return Ok(CheckStorageResponse {
300                include_topic: true,
301                messages: topic_resp.messages,
302                timestamp: topic_resp.timestamp,
303            });
304        }
305        // No topic messages, so carry on with normal ones, starting from the latest timestamp.
306        let timestamp = if self.flags.include_topic {
307            // See above, but Bigtable doesn't return the last message read timestamp when polling
308            // for topic messages. Instead, we'll use the explicitly set one we store in the User
309            // record and copy into the WebPushClient struct.
310            topic_resp.timestamp.or(self.current_timestamp)
311        } else {
312            timestamp
313        };
314        trace!(
315            "🗄️ WebPushClient::do_check_storage: fetch_timestamp_messages timestamp: {:?}",
316            timestamp
317        );
318        #[allow(unused_mut)]
319        let mut timestamp_resp = self
320            .app_state
321            .db
322            .fetch_timestamp_messages(&self.uaid, timestamp, 10)
323            .await?;
324        if !timestamp_resp.messages.is_empty() {
325            trace!(
326                "🗄️ WebPushClient::do_check_storage: Timestamp message returns: {:#?}",
327                timestamp_resp.messages
328            );
329            self.app_state
330                .metrics
331                .count_with_tags(
332                    "notification.message.retrieved",
333                    timestamp_resp.messages.len() as i64,
334                )
335                .with_tag("topic", "false")
336                .send();
337            #[cfg(feature = "reliable_report")]
338            // Since we pulled these from storage, mark them as "retrieved"
339            self.record_state(
340                &mut timestamp_resp.messages,
341                autopush_common::reliability::ReliabilityState::Retrieved,
342            )
343            .await;
344        }
345
346        Ok(CheckStorageResponse {
347            include_topic: false,
348            messages: timestamp_resp.messages,
349            // If we didn't get a timestamp off the last query, use the
350            // original value if passed one
351            timestamp: timestamp_resp.timestamp.or(timestamp),
352        })
353    }
354
355    /// Update the user's last Message read timestamp (for timestamp Messages)
356    ///
357    /// Called when a Client's Ack'd all timestamp messages sent to it to move
358    /// the timestamp Messages' "pointer". See
359    /// `AckState::unacked_stored_highest` for further information.
360    pub(super) async fn increment_storage(&mut self) -> Result<(), SMError> {
361        trace!(
362            "🗄️ WebPushClient::increment_storage: unacked_stored_highest: {:?}",
363            self.ack_state.unacked_stored_highest
364        );
365        let Some(timestamp) = self.ack_state.unacked_stored_highest else {
366            return Err(SMErrorKind::Internal(
367                "increment_storage w/ no unacked_stored_highest".to_owned(),
368            )
369            .into());
370        };
371        self.current_timestamp = Some(timestamp);
372        self.app_state
373            .db
374            .increment_storage(&self.uaid, timestamp)
375            .await?;
376        // Storage has advanced past these messages, so they won't be read
377        // again: record their outcomes
378        let acked = mem::take(&mut self.ack_state.acked_stored_timestamp_notifs);
379        for acked_notif in &acked {
380            self.record_delivered(
381                &acked_notif.notif,
382                DeliveryRoute::Stored,
383                acked_notif.ack_ms,
384            );
385        }
386        let expired = mem::take(&mut self.ack_state.expired_stored_timestamp_count);
387        emit_outcomes(
388            &self.app_state.metrics,
389            DeliveryPath::Ws,
390            Outcome::Expired,
391            expired,
392        );
393        #[cfg(feature = "reliable_report")]
394        {
395            let mut notifs = acked.into_iter().map(|acked| acked.notif).collect();
396            self.record_state(
397                &mut notifs,
398                autopush_common::reliability::ReliabilityState::Delivered,
399            )
400            .await;
401        }
402        self.flags.increment_storage = false;
403        Ok(())
404    }
405
406    /// Ensure this user hasn't exceeded the maximum allowed number of messages
407    /// read from storage (`Settings::msg_limit`)
408    ///
409    /// Drops the user record and returns the `SMErrorKind::UaidReset` error if
410    /// they have
411    async fn check_msg_limit(&mut self) -> Result<(), SMError> {
412        trace!(
413            "WebPushClient::check_msg_limit: sent_from_storage: {} msg_limit: {}",
414            self.sent_from_storage, self.app_state.settings.msg_limit
415        );
416        if self.sent_from_storage > self.app_state.settings.msg_limit {
417            // Exceeded the max limit of stored messages: drop the user to
418            // trigger a re-register
419            self.app_state
420                .metrics
421                .incr_with_tags(MetricName::UaExpiration)
422                .with_tag("reason", "too_many_messages")
423                .send();
424            self.app_state.db.remove_user(&self.uaid).await?;
425            return Err(SMErrorKind::UaidReset.into());
426        }
427        Ok(())
428    }
429
430    /// Emit metrics for a Notification to be sent to the user
431    fn emit_send_metrics(&self, notif: &Notification, source: SendSource) {
432        let metrics = &self.app_state.metrics;
433        let ua_info = &self.ua_info;
434        metrics
435            .incr_with_tags(MetricName::UaNotificationSent)
436            .with_tag("source", source.as_tag())
437            .with_tag("topic", &notif.topic.is_some().to_string())
438            .with_tag("os", &ua_info.metrics_os)
439            // TODO: include `internal` if meta is set
440            .send();
441        metrics
442            .count_with_tags(
443                "ua.message_data",
444                notif.data.as_ref().map_or(0, |data| data.len() as i64),
445            )
446            .with_tag("source", source.as_tag())
447            .with_tag("os", &ua_info.metrics_os)
448            .send();
449        // For messages pulled from storage, record how long they were stored before delivery.
450        // Don't record for direct sends (never entered storage)
451        if source == SendSource::Stored {
452            let stored_ms = sec_since_epoch().saturating_sub(notif.timestamp) * 1000;
453            metrics
454                .time_with_tags(MetricName::NotificationStorageTime.as_ref(), stored_ms)
455                .with_tag("topic", &notif.topic.is_some().to_string())
456                .with_tag("os", &ua_info.metrics_os)
457                .send();
458        }
459    }
460}