Skip to main content

autopush_common/
delivery_metrics.rs

1//! Aggregate delivery latency and outcome metrics.
2//!
3//! These answer "is push slow?" and "is push reliable?" for all traffic without
4//! tracking individual messages. Latency is recorded in Prometheus histograms
5//! (served from `/metrics`) so quantiles can be aggregated across pods and time
6//! windows. Terminal outcomes are statsd counters (`notification.outcome`).
7use cadence::{Counted, StatsdClient};
8use prometheus_client::{
9    encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder, text::encode},
10    metrics::{family::Family, histogram::Histogram},
11    registry::{Registry, Unit},
12};
13
14use crate::metric_name::MetricName;
15
16/// Latency histogram bucket boundaries, in seconds.
17///
18/// Log-spaced from 10ms to the 30 day maximum notification TTL, so both direct
19/// delivery (ms) and offline devices (hours/days) are resolvable. Both
20/// autoendpoint and autoconnect use these so the series are comparable.
21/// Changing these breaks historical comparison.
22pub const DELIVERY_BUCKETS: [f64; 24] = [
23    0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 300.0, 900.0, 1800.0, 3600.0,
24    10800.0, 21600.0, 43200.0, 86400.0, 172800.0, 345600.0, 604800.0, 1209600.0, 2592000.0,
25];
26
27/// Content type for the `/metrics` response.
28pub const CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
29
30/// The destination router of a notification.
31#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
32pub enum DeliveryPath {
33    /// Delivered over a WebSocket by autoconnect.
34    Ws,
35    Fcm,
36    Apns,
37    /// The router could not be determined (e.g. the subscription could not be
38    /// decoded). Only used for `notification.outcome`.
39    Unknown,
40}
41
42impl DeliveryPath {
43    pub fn as_str(&self) -> &'static str {
44        match self {
45            DeliveryPath::Ws => "ws",
46            DeliveryPath::Fcm => "fcm",
47            DeliveryPath::Apns => "apns",
48            DeliveryPath::Unknown => "unknown",
49        }
50    }
51}
52
53impl EncodeLabelValue for DeliveryPath {
54    fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> {
55        std::fmt::Write::write_str(encoder, self.as_str())
56    }
57}
58
59/// Whether the notification was read from storage before delivery.
60#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
61pub enum DeliveryRoute {
62    Direct,
63    /// Read from storage before delivery, including direct deliveries that
64    /// were saved because the client disconnected before ACKing.
65    Stored,
66}
67
68impl DeliveryRoute {
69    pub fn as_str(&self) -> &'static str {
70        match self {
71            DeliveryRoute::Direct => "direct",
72            DeliveryRoute::Stored => "stored",
73        }
74    }
75}
76
77impl EncodeLabelValue for DeliveryRoute {
78    fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> {
79        std::fmt::Write::write_str(encoder, self.as_str())
80    }
81}
82
83/// Who is responsible for a non-delivery.
84#[derive(Clone, Copy, Debug, PartialEq, Eq)]
85pub enum Cause {
86    None,
87    Client,
88    Bridge,
89    Push,
90}
91
92impl Cause {
93    pub fn as_str(&self) -> &'static str {
94        match self {
95            Cause::None => "none",
96            Cause::Client => "client",
97            Cause::Bridge => "bridge",
98            Cause::Push => "push",
99        }
100    }
101}
102
103/// The terminal status of a notification.
104#[derive(Clone, Copy, Debug, PartialEq, Eq)]
105pub enum Outcome {
106    /// Received by the client (any ACK code) or accepted by the bridge.
107    Delivered,
108    /// TTL=0 and not directly deliverable, or a stored message that had
109    /// expired by the time it was read. Messages that expire without ever
110    /// being read are not counted.
111    Expired,
112    /// 404, e.g. the endpoint token could not be decrypted.
113    NotFound,
114    /// 410, the subscription or user no longer exists.
115    Gone,
116    /// Any other FCM/APNs error.
117    BridgeError,
118    /// The notification could not be saved to storage.
119    StorageError,
120    /// Any other 5xx.
121    InternalError,
122}
123
124impl Outcome {
125    pub fn as_str(&self) -> &'static str {
126        match self {
127            Outcome::Delivered => "delivered",
128            Outcome::Expired => "expired",
129            Outcome::NotFound => "not_found",
130            Outcome::Gone => "gone",
131            Outcome::BridgeError => "bridge_error",
132            Outcome::StorageError => "storage_error",
133            Outcome::InternalError => "internal_error",
134        }
135    }
136
137    pub fn cause(&self) -> Cause {
138        match self {
139            Outcome::Delivered | Outcome::Expired => Cause::None,
140            Outcome::NotFound | Outcome::Gone => Cause::Client,
141            Outcome::BridgeError => Cause::Bridge,
142            Outcome::StorageError | Outcome::InternalError => Cause::Push,
143        }
144    }
145}
146
147/// Increment `notification.outcome`.
148pub fn emit_outcome(metrics: &StatsdClient, path: DeliveryPath, outcome: Outcome) {
149    emit_outcomes(metrics, path, outcome, 1);
150}
151
152/// Add `count` to `notification.outcome`.
153pub fn emit_outcomes(metrics: &StatsdClient, path: DeliveryPath, outcome: Outcome, count: i64) {
154    if count == 0 {
155        return;
156    }
157    metrics
158        .count_with_tags(MetricName::NotificationOutcome.as_ref(), count)
159        .with_tag("path", path.as_str())
160        .with_tag("outcome", outcome.as_str())
161        .with_tag("cause", outcome.cause().as_str())
162        .send();
163}
164
165#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
166struct DeliveryLabels {
167    path: DeliveryPath,
168    route: DeliveryRoute,
169}
170
171fn delivery_histogram() -> Histogram {
172    Histogram::new(DELIVERY_BUCKETS)
173}
174
175/// Seconds elapsed between two millisecond timestamps. Negative durations
176/// (cross-host clock skew) are clamped to 0.
177fn elapsed_secs(start_ms: u64, end_ms: u64) -> f64 {
178    end_ms.saturating_sub(start_ms) as f64 / 1000.0
179}
180
181/// The process wide Prometheus registry and the histograms it holds.
182#[derive(Debug)]
183pub struct DeliveryMetrics {
184    registry: Registry,
185    delivery_seconds: Family<DeliveryLabels, Histogram, fn() -> Histogram>,
186    stored_delivery_after_connect_seconds: Histogram,
187    stored_offline_seconds: Histogram,
188    /// The legacy reliability state gauges, updated on each `/metrics` request.
189    /// Only registered by autoendpoint (see `with_reliability_report`).
190    #[cfg(feature = "reliable_report")]
191    pub reliability_report: Option<crate::reliability::ReportFamily>,
192}
193
194impl Default for DeliveryMetrics {
195    fn default() -> Self {
196        Self::new()
197    }
198}
199
200impl DeliveryMetrics {
201    pub fn new() -> Self {
202        Self::build(false)
203    }
204
205    /// Also register the legacy reliability state gauges. Only autoendpoint
206    /// serves these, since only it updates them.
207    #[cfg(feature = "reliable_report")]
208    pub fn with_reliability_report() -> Self {
209        Self::build(true)
210    }
211
212    #[cfg_attr(not(feature = "reliable_report"), allow(unused_variables))]
213    fn build(with_reliability_report: bool) -> Self {
214        let mut registry = Registry::default();
215        let delivery_seconds =
216            Family::<DeliveryLabels, Histogram, fn() -> Histogram>::new_with_constructor(
217                delivery_histogram,
218            );
219        registry.register_with_unit(
220            "autopush_delivery",
221            "Time from receipt at autoendpoint to delivery",
222            Unit::Seconds,
223            delivery_seconds.clone(),
224        );
225        let stored_delivery_after_connect_seconds = delivery_histogram();
226        registry.register_with_unit(
227            "autopush_stored_delivery_after_connect",
228            "Time to deliver a stored message once the client was connected",
229            Unit::Seconds,
230            stored_delivery_after_connect_seconds.clone(),
231        );
232        let stored_offline_seconds = delivery_histogram();
233        registry.register_with_unit(
234            "autopush_stored_offline",
235            "Time a stored message waited for the client to connect",
236            Unit::Seconds,
237            stored_offline_seconds.clone(),
238        );
239        #[cfg(feature = "reliable_report")]
240        let reliability_report =
241            with_reliability_report.then(|| crate::reliability::register_report(&mut registry));
242        Self {
243            registry,
244            delivery_seconds,
245            stored_delivery_after_connect_seconds,
246            stored_offline_seconds,
247            #[cfg(feature = "reliable_report")]
248            reliability_report,
249        }
250    }
251
252    /// Record `autopush_delivery_seconds`.
253    pub fn observe_delivery(
254        &self,
255        path: DeliveryPath,
256        route: DeliveryRoute,
257        start_ms: u64,
258        end_ms: u64,
259    ) {
260        self.delivery_seconds
261            .get_or_create(&DeliveryLabels { path, route })
262            .observe(elapsed_secs(start_ms, end_ms));
263    }
264
265    /// Record the two parts of a stored message's delivery, split at when it
266    /// became deliverable (the later of the client connecting and the message
267    /// being received):
268    ///
269    /// - `autopush_stored_offline_seconds`: receipt to deliverable, the time
270    ///   spent waiting for the client to connect.
271    /// - `autopush_stored_delivery_after_connect_seconds`: deliverable to ACK.
272    pub fn observe_stored_delivery(&self, connected_at_ms: u64, received_ms: u64, ack_ms: u64) {
273        let deliverable_ms = connected_at_ms.max(received_ms);
274        self.stored_offline_seconds
275            .observe(elapsed_secs(received_ms, deliverable_ms));
276        self.stored_delivery_after_connect_seconds
277            .observe(elapsed_secs(deliverable_ms, ack_ms));
278    }
279
280    /// Encode the registry in the OpenMetrics text format.
281    pub fn encode(&self) -> Result<String, std::fmt::Error> {
282        let mut encoded = String::new();
283        encode(&mut encoded, &self.registry)?;
284        Ok(encoded)
285    }
286}
287
288#[cfg(test)]
289mod tests {
290    use super::*;
291
292    #[test]
293    fn buckets_are_sorted() {
294        assert!(DELIVERY_BUCKETS.windows(2).all(|w| w[0] < w[1]));
295        assert_eq!(
296            *DELIVERY_BUCKETS.last().unwrap(),
297            crate::MAX_NOTIFICATION_TTL_SECS as f64
298        );
299    }
300
301    #[test]
302    fn clock_skew_clamped() {
303        assert_eq!(elapsed_secs(2_000, 1_000), 0.0);
304        assert_eq!(elapsed_secs(1_000, 3_500), 2.5);
305    }
306
307    #[test]
308    fn encode_report() {
309        let metrics = DeliveryMetrics::new();
310        metrics.observe_delivery(DeliveryPath::Ws, DeliveryRoute::Direct, 1_000, 1_030);
311        metrics.observe_delivery(DeliveryPath::Fcm, DeliveryRoute::Direct, 2_000, 1_000);
312        metrics.observe_stored_delivery(10_000, 5_000, 12_000);
313        let report = metrics.encode().unwrap();
314        assert!(report.contains("# TYPE autopush_delivery_seconds histogram"));
315        assert!(report.contains("# UNIT autopush_delivery_seconds seconds"));
316        assert!(
317            report.contains(
318                r#"autopush_delivery_seconds_bucket{le="0.05",path="ws",route="direct"} 1"#
319            )
320        );
321        assert!(report.contains(
322            r#"autopush_delivery_seconds_bucket{le="0.01",path="fcm",route="direct"} 1"#
323        ));
324        assert!(report.contains(r#"autopush_delivery_seconds_count{path="ws",route="direct"} 1"#));
325        // 12s - max(10s, 5s) = 2s
326        assert!(report.contains("autopush_stored_delivery_after_connect_seconds_sum 2.0"));
327        // max(10s, 5s) - 5s = 5s
328        assert!(report.contains("autopush_stored_offline_seconds_sum 5.0"));
329        assert!(report.trim_end().ends_with("# EOF"));
330    }
331
332    #[test]
333    fn reliability_report_is_opt_in() {
334        let report = DeliveryMetrics::new().encode().unwrap();
335        assert!(!report.contains("autopush_reliability"));
336        #[cfg(feature = "reliable_report")]
337        {
338            assert!(DeliveryMetrics::new().reliability_report.is_none());
339            let metrics = DeliveryMetrics::with_reliability_report();
340            crate::reliability::update_report(
341                metrics.reliability_report.as_ref().unwrap(),
342                std::collections::HashMap::from([("received".to_owned(), 5)]),
343            );
344            let report = metrics.encode().unwrap();
345            assert!(report.contains(r#"autopush_reliability{state="received"} 5"#));
346        }
347    }
348
349    #[test]
350    fn outcome_causes() {
351        assert_eq!(Outcome::Delivered.cause(), Cause::None);
352        assert_eq!(Outcome::Expired.cause(), Cause::None);
353        assert_eq!(Outcome::NotFound.cause(), Cause::Client);
354        assert_eq!(Outcome::Gone.cause(), Cause::Client);
355        assert_eq!(Outcome::BridgeError.cause(), Cause::Bridge);
356        assert_eq!(Outcome::StorageError.cause(), Cause::Push);
357        assert_eq!(Outcome::InternalError.cause(), Cause::Push);
358    }
359}