Skip to main content

autopush_common/
reliability.rs

1/// Push Reliability Recorder
2///
3/// This allows us to track messages from select, known parties (currently, just
4/// mozilla generated and consumed) so that we can identify potential trouble spots
5/// and where messages expire early. Message expiration can lead to message loss
6use std::collections::HashMap;
7use std::str::FromStr;
8use std::sync::Arc;
9
10use cadence::StatsdClient;
11use chrono::TimeDelta;
12use deadpool_redis::Config;
13
14use prometheus_client::{metrics::family::Family, metrics::gauge::Gauge, registry::Registry};
15use redis::{AsyncCommands, aio::ConnectionLike};
16use redis::{Pipeline, Value};
17
18use crate::db::client::DbClient;
19use crate::errors::{ApcError, ApcErrorKind, Result};
20use crate::metric_name::MetricName;
21use crate::metrics::StatsdClientExt;
22use crate::util::timing::sec_since_epoch;
23
24// Redis Keys
25pub const COUNTS: &str = "state_counts";
26pub const EXPIRY: &str = "expiry";
27
28const CONNECTION_EXPIRATION: TimeDelta = TimeDelta::seconds(10);
29// Minimum expiration period of 1 second.
30// This was set to `0`, but there was some confusion whether that would not set an
31// expiration time for a record or would set a record not to expire.
32const MIN_EXPIRATION: u64 = 1;
33
34/// The various states that a message may transit on the way from reception to delivery.
35// Note: "Message" in this context refers to the Subscription Update.
36// TODO: Differentiate between "transmitted via webpush" and "transmitted via bridge"?
37#[derive(
38    Debug,
39    Clone,
40    Copy,
41    PartialEq,
42    Eq,
43    serde::Deserialize,
44    serde::Serialize,
45    strum::Display,
46    strum::EnumString,
47)]
48#[serde(rename_all = "snake_case")]
49#[strum(serialize_all = "snake_case")]
50pub enum ReliabilityState {
51    Received,          // Message was received by the Push Server
52    Stored,            // Message was stored because it could not be delivered immediately
53    Retrieved,         // Message was taken from storage for delivery
54    IntTransmitted,    // Message was handed off between autoendpoint and autoconnect
55    IntAccepted,       // Message was accepted by autoconnect from autopendpoint
56    BridgeTransmitted, // Message was handed off to a mobile bridge for eventual delivery
57    Transmitted,       // Message was handed off for delivery to the UA
58    Accepted,          // Message was accepted for delivery by the UA
59    Delivered,         // Message was provided to the WebApp recipient by the UA
60    DecryptionError,   // Message was provided to the UA and it reported a decryption error
61    NotDelivered,      // Message was provided to the UA and it reported a not delivered error
62    Expired,           // Message expired naturally (e.g. TTL=0)
63    Errored,           // Message resulted in an Error state and failed to be delivered.
64}
65
66impl ReliabilityState {
67    ///  Has the message reached a state where no further transitions should be possible?
68    pub fn is_terminal(self) -> bool {
69        // NOTE: this list should match the daily snapshot captured by `reliability_cron.py`
70        // which will trim these counts after the max message TTL has expired.
71        matches!(
72            self,
73            ReliabilityState::DecryptionError
74                | ReliabilityState::BridgeTransmitted
75                | ReliabilityState::Delivered
76                | ReliabilityState::Errored
77                | ReliabilityState::Expired
78                | ReliabilityState::NotDelivered
79        )
80    }
81}
82
83#[derive(Clone)]
84pub struct PushReliability {
85    pool: Option<deadpool_redis::Pool>,
86    db: Box<dyn DbClient>,
87    metrics: Arc<StatsdClient>,
88    retries: usize,
89}
90
91// Define a struct to hold the expiry key, since it's easy to flub the order.
92pub struct ExpiryKey {
93    pub id: String,
94    pub state: ReliabilityState,
95}
96
97impl std::fmt::Display for ExpiryKey {
98    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
99        write!(f, "{}#{}", self.id, self.state)
100    }
101}
102
103impl TryFrom<String> for ExpiryKey {
104    type Error = ApcError;
105    fn try_from(value: String) -> Result<Self> {
106        let (id, state) = value.split_once('#').ok_or_else(|| {
107            ApcErrorKind::GeneralError("ExpiryKey must be in the format 'id#state'".to_owned())
108        })?;
109        let state: ReliabilityState = ReliabilityState::from_str(state).map_err(|_| {
110            ApcErrorKind::GeneralError(
111                "Invalid state in ExpiryKey, must be a valid ReliabilityState".to_owned(),
112            )
113        })?;
114        Ok(Self {
115            id: id.to_owned(),
116            state,
117        })
118    }
119}
120
121impl PushReliability {
122    // Do the magic to make a report instance, whatever that will be.
123    pub fn new(
124        reliability_dsn: &Option<String>,
125        db: Box<dyn DbClient>,
126        metrics: &Arc<StatsdClient>,
127        retries: usize,
128    ) -> Result<Self> {
129        let Some(reliability_dsn) = reliability_dsn else {
130            debug!("πŸ” No reliability DSN declared.");
131            return Ok(Self {
132                pool: None,
133                db: db.clone(),
134                metrics: metrics.clone(),
135                retries,
136            });
137        };
138
139        let config: deadpool_redis::Config = Config::from_url(reliability_dsn);
140        let pool = Some(
141            config
142                .builder()
143                .map_err(|e| {
144                    ApcErrorKind::GeneralError(format!("Could not config reliability pool {e:?}"))
145                })?
146                .create_timeout(Some(CONNECTION_EXPIRATION.to_std().unwrap()))
147                .runtime(deadpool::Runtime::Tokio1)
148                .build()
149                .map_err(|e| {
150                    ApcErrorKind::GeneralError(format!("Could not build reliability pool {e:?}"))
151                })?,
152        );
153
154        Ok(Self {
155            pool,
156            db: db.clone(),
157            metrics: metrics.clone(),
158            retries,
159        })
160    }
161
162    // Record the message state change to storage.
163    pub async fn record(
164        &self,
165        reliability_id: &Option<String>,
166        new: ReliabilityState,
167        old: &Option<ReliabilityState>,
168        expr: Option<u64>,
169    ) -> Result<Option<ReliabilityState>> {
170        let Some(id) = reliability_id else {
171            return Ok(None);
172        };
173        if let Some(pool) = &self.pool {
174            debug!(
175                "πŸ” {} from {} to {}",
176                id,
177                old.map(|v| v.to_string())
178                    .unwrap_or_else(|| "None".to_owned()),
179                new
180            );
181            match pool.get().await {
182                Ok(mut conn) => {
183                    self.internal_record(&mut conn, old, new, expr, id)
184                        .await
185                        .map_err(|e| {
186                            warn!("πŸ”βš οΈ Unable to record reliability state: {:?}", e);
187                            ApcErrorKind::GeneralError(
188                                "Could not record reliability state".to_owned(),
189                            )
190                        })?;
191                }
192                Err(e) => warn!("πŸ”βš οΈ Unable to get reliability state pool, {:?}", e),
193            };
194        };
195        // Errors are not fatal, and should not impact message flow, but
196        // we should record them somewhere.
197        let _ = self.db.log_report(id, new).await.inspect_err(|e| {
198            warn!("πŸ”βš οΈ Unable to record reliability state log: {:?}", e);
199        });
200        Ok(Some(new))
201    }
202
203    /// Record the state change in the reliability datastore.
204    /// Because of a strange duplication error, we're using three "tables".
205    /// The COUNTS table contains the count of messages in each state. It's provided as a quick lookup
206    /// for the dashboard query.
207    /// The EXPIRY table contains the expiration time for messages. Those are cleaned up by the
208    /// `gc` function.
209    /// Finally, there's the "table" that is `state.{id}`. This contains the current state of each
210    /// message. Ideally, this would use something like `HSETEX`, but that is not available until
211    /// Redis 8.0, so for now, we use the `SET` function, which provides an expiration time.
212    /// BE SURE TO SET AN EXPIRATION TIME FOR EACH STATE MESSAGE!
213    // TODO: Do we need to keep the state in the message? Probably good for sanity reasons.
214    pub(crate) async fn internal_record<C: ConnectionLike + AsyncCommands>(
215        &self,
216        conn: &mut C,
217        old: &Option<ReliabilityState>,
218        new: ReliabilityState,
219        expr: Option<u64>,
220        id: &str,
221    ) -> Result<()> {
222        trace!(
223            "πŸ” internal record: {} from {} to {}",
224            id,
225            old.map(|v| v.to_string())
226                .unwrap_or_else(|| "None".to_owned()),
227            new
228        );
229
230        let state_key = format!("state.{id}");
231        trace!("πŸ” state key: {}", &state_key);
232        /*
233        ## Reliability lock and adjustment
234
235        This transaction creates a lock on the individual messages "state_key" (which contains the current messages state)
236        and the entire Expiration table. If either of those values are changed while the transaction is in progress, then
237        the transaction will fail, return a `Redis::Nil`, and the transaction will retry up to `self.retries` (Note:
238        none of the operations in the transaction will have taken place yet.)
239
240        We want to lock on the `state_key` because that shows the given state of the message.
241        We want to lock on the `expiry` table because the message may have expired, been handled by the `gc()` function and
242        may have already been adjusted. (Remember, the `expiry` table holds timestamp markers for when a message will be
243        expiring.)
244
245        The operations in the Pipeline are:
246
247        1. If there is an `old` state, decrement the old state count and remove the old marker from the `expiry` table.
248        2. If the `new` state is not a terminal state (e.g. it is not a final disposition for a message), then create a new
249           entry for `expiry`
250        3. Increment the `new` state count
251        4. Modify the `state.{ID}` value (if it exists) to indicate the newest known state for the message, returning the
252           prior value. Checking that this value is not Nil is an additional sanity check that the value was not removed
253           or altered by a different process. (We are already imposing a transaction lock which should prevent this, but
254           additional paranoia can sometimes be good.)
255
256        Upon execution of the Pipeline, we get back a list of results. This should look similar to a `Queued` entry for
257        every function that we've performed, followed by an inline list of the results of those functions. (e.g.
258        for a transition between `ReliabilityState::Stored` to `ReliabilityState::Retrieved`, which are neither terminal
259        states), we should see something like: `["Queued","Queued","Queued",["OK", "OK", "stored"]]`
260
261        If any of the locks failed, we should get back a `Nil`, in which case, we would want to try this operation again.
262        In addition, the sanity `Get` may also return a `Nil` which would also indicate that there was a problem. There's
263        some debate whether or not to retry the operation if that's the case (since it would imply that the lock failed
264        for some unexpected reason), however for now, we just report a soft `error!()`.
265
266         */
267        // Create the initial state key.
268        // Do not place this inside of the transaction. We monitor the state key and the transaction will
269        // fail because the value will change before the transaction completes. Yes, really.
270        if new == ReliabilityState::Received && old.is_none() {
271            trace!(
272                "πŸ” Creating new record {state_key} ex {:?}",
273                expr.unwrap_or(MIN_EXPIRATION)
274            );
275            // we can't perform this in a transaction because we can only increment if the set succeeds,
276            // and values aren't returned when creating values in transactions. In order to do this
277            // from inside the transaction, you would need to create a function, and that feels a bit
278            // too heavy for this.
279            // Create the new `state.{id}` key if it does not exist, and set the expiration.
280            let options = redis::SetOptions::default()
281                .with_expiration(redis::SetExpiry::EX(expr.unwrap_or(MIN_EXPIRATION)))
282                .conditional_set(redis::ExistenceCheck::NX);
283            trace!("πŸ” β­• SET {state_key} NX EX {:?}", new);
284            let result = conn
285                .set_options::<_, _, Value>(&state_key, new.to_string(), options)
286                .await
287                .map_err(|e| {
288                    //dbg!(&e);
289                    warn!("πŸ”βš οΈ Could not create state key: {:?}", e);
290                    ApcErrorKind::GeneralError("Could not create the state key".to_owned())
291                })?;
292            if result != redis::Value::Okay {
293                // Redis returned a `Nil`, indicating that there was some error. The only thing that should cause that
294                // would be if the `old` state was reset to `None` and we thought we needed to create a new state.
295                // Since the message carries it's prior state, it shouldn't be set to `None` unless there's something
296                // strange going on.
297                // TODO: It's worth noting that when restarting autoendpoint, we get a large number of these in the logs.
298                // Need to figure out the reason for that.
299                // The `result` is always `nil` so that's not helpful.
300                error!("πŸ”βš οΈ Tried to recreate state_key {state_key}: {old:?} => {new:?}",);
301                return Err(
302                    ApcErrorKind::GeneralError("Tried to recreate state_key".to_string()).into(),
303                );
304            }
305        } else {
306            trace!("πŸ” Checking {:?}", &old);
307            // safety check (yes, there's still a slight chance of a race, but it's small)
308            if let Some(old) = old {
309                let check_state: String = conn.get(&state_key).await?;
310                trace!("πŸ” Checking state for {}: {:?}", id, &check_state);
311                if check_state != old.to_string() {
312                    trace!(
313                        "πŸ” Attempting to update state for {} from {} to {}, but current state is different: {:?}",
314                        id, old, new, check_state
315                    );
316                    return Err(ApcErrorKind::GeneralError(
317                        "State mismatch during reliability record update".to_owned(),
318                    )
319                    .into());
320                }
321            };
322        }
323
324        crate::redis_util::transaction(
325            conn,
326            &[&state_key, &EXPIRY.to_owned()],
327            self.retries,
328            || ApcErrorKind::GeneralError("Exceeded reliability record retry attempts".to_owned()),
329            async |conn, pipe: &mut Pipeline| {
330                // The first state is special, since we need to create the `state_key`.
331                // remove the old state from the expiry set, if it exists.
332                // There should only be one message at a given state in the `expiry` table.
333                // Since we only use that table to track messages that may expire. (We
334                // decrement "expired" messages in the `gc` function, so having messages
335                // in multiple states may decrement counts incorrectly.))
336                if let Some(old) = old {
337                    trace!("πŸ” βž– {old} - {id}");
338                    trace!("πŸ” πŸͺˆβ­• HINCRBY {COUNTS} 1");
339                    pipe.hincr(COUNTS, old.to_string(), -1);
340                    let key = ExpiryKey {
341                        id: id.to_string(),
342                        state: old.to_owned(),
343                    }
344                    .to_string();
345                    trace!("πŸ” internal remove old state: {:?}", key);
346                    trace!("πŸ” πŸͺˆβ­• ZREM {EXPIRY} {key}");
347                    pipe.zrem(EXPIRY, &key);
348                }
349                if !new.is_terminal() {
350                    // Write the expiration only if the state is non-terminal. Otherwise we run the risk of
351                    // messages reporting a false "expired" state even if they were "successful".
352                    let key = ExpiryKey {
353                        id: id.to_string(),
354                        state: new.to_owned(),
355                    }
356                    .to_string();
357                    trace!("πŸ” πŸͺˆβ­• ZADD {EXPIRY} {:?} {key}", expr.unwrap_or_default());
358                    pipe.zadd(EXPIRY, &key, expr.unwrap_or_default());
359                    trace!("πŸ” internal record result: {:?}", key);
360                }
361                trace!("πŸ” βž• {new} - {id}");
362                // Bump up the new state count, and set the state key's state if it still exists.
363                trace!("πŸ” πŸͺˆβ­• HINCRBY {COUNTS} {new} 1");
364                pipe.hincr(COUNTS, new.to_string(), 1);
365                if new != ReliabilityState::Received {
366                    let options = redis::SetOptions::default()
367                        .with_expiration(redis::SetExpiry::KEEPTTL)
368                        .conditional_set(redis::ExistenceCheck::XX)
369                        .get(true);
370                    trace!("πŸ” πŸͺˆβ­• SET {state_key} {new} XX KEEPTTL");
371                    pipe.set_options(&state_key, new.to_string(), options);
372                }
373                // `exec_query` returns `RedisResult<()>`.
374                // `query_async` returns `RedisResult<Option<redis::Value>, RedisError>`.
375                // We really don't care about the returned result here, but transaction
376                // retries if we return Ok(None), so we run the exec and return
377                // a nonce `Some` value.
378                // The turbo-fish is a fallback for edition 2024
379                let result = pipe.query_async::<redis::Value>(conn).await?;
380                trace!("πŸ” πŸͺˆ {id} - {:?}", &result);
381                // The last element returned from the command is the result of the pipeline.
382                // If Redis encounters an error, it will return a `nil` as well. We handle both
383                // the same (retry), so we can normalize errors as `nil`.
384                // The last of which should be the result of the piped command set.
385                // This should return `nil` if there is any error, in which case we record
386                // a soft error. (On success, it will return the result of the last command
387                // in the pipe, which may vary due to the current state).
388                // This could also be strung together as a cascade of functions, but it's broken
389                // out to discrete steps for readability.
390                /* On prod, we get a large number of these errors, which I think might be clogging
391                  up the redis connections, causing servers to report as degraded.
392                */
393                if result == redis::Value::Nil {
394                    warn!("πŸ”βš πŸͺˆ {id} - Pipe failed, skipping retry.");
395                    // temporarily just let things fail to handle autoendpoint degradation.
396                    // return Ok(None);
397                    return Ok(Some(redis::Value::Okay));
398                }
399                if let Some(operations) = result.as_sequence() {
400                    // We have responses, the first items report the state of the commands,
401                    // the final line is a list of command results.
402                    if let Some(result_values) = operations.last() && let Some(results) = result_values.as_sequence() &&
403                            // The last command should contain the prior state. If it returned `Nil`
404                            // for some, unexpected reason, note the error.
405                            Some(&redis::Value::Nil) == results.last() {
406                                error!("πŸ”πŸš¨ WARNING: Lock Issue for {id}")
407                                // There is some debate about whether or not to rerun
408                                // the transaction if this state is reached.
409                                // Rerunning would cause the counts to be impacted, but
410                                // might address any other issues that caused the
411                                // `Nil` to be returned.
412                                // For now, let's just log the error.
413
414
415                    }
416                    Ok(Some(redis::Value::Okay))
417                } else {
418                    // a Nil will rerun the transaction.
419                    Ok(None)
420                }
421            },
422        )
423        .await
424        .inspect_err(|e| {
425            //dbg!(&e);
426            warn!("πŸ”βš οΈError occurred during transaction: {:?}", e);
427        })?;
428        Ok(())
429    }
430
431    /// Perform a garbage collection cycle on a reliability object.
432    pub async fn gc(&self) -> Result<()> {
433        if let Some(pool) = &self.pool
434            && let Ok(mut conn) = pool.get().await
435        {
436            debug!("πŸ” performing pre-report garbage collection");
437            return self.internal_gc(&mut conn, sec_since_epoch()).await;
438        }
439        Ok(())
440    }
441
442    // Perform the `garbage collection` cycle. This will scan the currently known timestamp
443    // indexed entries in redis looking for "expired" data, and then rectify the counts to
444    // indicate the final states. This is because many of the storage systems do not provide
445    // indicators when data reaches a TTL.
446
447    // A few notes about redis:
448    // "pipeline" essentially stitches commands together. Each command executes in turn, but the data store
449    // remains open for other operations to occur.
450    // `atomic()` wraps things in a transaction, essentially locking the data store while the command executes.
451    // Sadly, there's no way to create a true transaction where you read and write in a single operation, so we
452    // have to presume some "slop" here.
453    pub(crate) async fn internal_gc<C: ConnectionLike + AsyncCommands>(
454        &self,
455        conn: &mut C,
456        expr: u64,
457    ) -> Result<()> {
458        let result: redis::Value = crate::redis_util::transaction(
459            conn,
460            &[EXPIRY],
461            self.retries,
462            || ApcErrorKind::GeneralError("Exceeded gc retry attempts".to_owned()),
463            async |conn, pipe| {
464                // First, get the list of values that are to be purged.
465                let purged: Vec<String> = conn.zrangebyscore(EXPIRY, 0, expr as isize).await?;
466                // insta-bail if there's nothing to do.
467                if purged.is_empty() {
468                    return Ok(Some(redis::Value::Nil));
469                }
470
471                // Now purge each of the values by resetting the counts and removing the item from
472                // the purge set.
473                for key in purged {
474                    let Ok(expiry_key) = ExpiryKey::try_from(key.clone()) else {
475                        let err = "Invalid key stored in Reliability datastore";
476                        error!("πŸ”πŸŸ₯ {} [{:?}]", &err, &key);
477                        return Err(ApcErrorKind::GeneralError(err.to_owned()));
478                    };
479                    // Adjust the COUNTS and then remove the record from the list of expired rows.
480                    trace!("πŸ” βž–πŸͺ¦ {} - {key}", expiry_key.state.to_string());
481                    pipe.hincr(COUNTS, expiry_key.state.to_string(), -1);
482                    pipe.hincr(COUNTS, ReliabilityState::Expired.to_string(), 1);
483                    pipe.zrem(EXPIRY, key);
484                }
485                Ok(pipe.query_async(conn).await?)
486            },
487        )
488        .await?;
489        self.metrics
490            .incr_with_tags(MetricName::ReliabilityGc)
491            .with_tag(
492                "status",
493                if result == redis::Value::Nil {
494                    "error"
495                } else {
496                    "success"
497                },
498            )
499            .send();
500        Ok(())
501    }
502
503    // Return a snapshot of milestone states
504    // This will probably not be called directly, but useful for debugging.
505    pub async fn report(&self) -> Result<HashMap<String, i32>> {
506        if let Some(pool) = &self.pool
507            && let Ok(mut conn) = pool.get().await
508        {
509            return Ok(conn.hgetall(COUNTS).await.map_err(|e| {
510                ApcErrorKind::GeneralError(format!("Could not read report {e:?}"))
511            })?);
512        }
513        Ok(HashMap::new())
514    }
515
516    pub async fn health_check<'a>(&self) -> Result<&'a str> {
517        if let Some(pool) = &self.pool {
518            let mut conn = pool.get().await.map_err(|e| {
519                ApcErrorKind::GeneralError(format!(
520                    "Could not connect to reliability datastore: {e:?}"
521                ))
522            })?;
523            // Add a type here, even though we're tossing the value, in order to prevent the `FromRedisValue` warning.
524            conn.ping::<()>().await.map_err(|e| {
525                ApcErrorKind::GeneralError(format!("Could not ping reliability datastore: {e:?}"))
526            })?;
527            Ok("OK")
528        } else {
529            Ok("OK")
530        }
531    }
532}
533
534const METRIC_NAME: &str = "autopush_reliability";
535
536/// A "family" is a grouping of metrics.
537/// we specify this as the ("label", "label value") which index to a Gauge.
538pub type ReportFamily = Family<Vec<(&'static str, String)>, Gauge>;
539
540/// Register the reliability state gauges with the given registry.
541pub fn register_report(registry: &mut Registry) -> ReportFamily {
542    let family = ReportFamily::default();
543    // This creates the top level association of the elements in the family with the metric.
544    registry.register(
545        METRIC_NAME,
546        "Count of messages at given states",
547        family.clone(),
548    );
549    family
550}
551
552/// Set the reliability state gauges from the given milestone counts.
553///
554/// The family is long lived, so it's cleared first: states missing from
555/// `values` (e.g. when Redis is unavailable) are dropped rather than left at
556/// their last known value.
557pub fn update_report(family: &ReportFamily, values: HashMap<String, i32>) {
558    family.clear();
559    for (milestone, value) in values.into_iter() {
560        // prevent any stray leakage of invalid state data
561        if ReliabilityState::from_str(&milestone).is_err() {
562            trace!("πŸ” skipping invalid state {milestone:?}");
563            continue;
564        }
565        // Specify the static "state" label name with the given milestone, and add the
566        // value as the gauge value.
567        family
568            .get_or_create(&vec![("state", milestone)])
569            .set(value.into());
570    }
571}
572
573#[cfg(test)]
574mod tests {
575    use std::collections::HashMap;
576    use std::str::FromStr;
577
578    use super::*;
579    use prometheus_client::encoding::text::encode;
580    use redis_test::{MockCmd, MockRedisConnection};
581    use uuid::Uuid;
582
583    #[test]
584    fn test_report() {
585        // create a nonce report
586        let mut report: HashMap<String, i32> = HashMap::new();
587        let recv = ReliabilityState::Received.to_string();
588        let trns = ReliabilityState::Transmitted.to_string();
589        report.insert(recv.clone(), 111);
590        report.insert(ReliabilityState::Stored.to_string(), 222);
591        report.insert(ReliabilityState::Retrieved.to_string(), 333);
592        report.insert(trns.clone(), 444);
593        report.insert("biginvalid".to_string(), -1);
594
595        let mut registry = Registry::default();
596        let family = register_report(&mut registry);
597        update_report(&family, report);
598        let mut generated = String::new();
599        encode(&mut generated, &registry).unwrap();
600        // We don't really care if the `Created` or `HELP` lines are included
601        assert!(generated.contains(&format!("# TYPE {METRIC_NAME}")));
602        // sample the first and last values.
603        assert!(generated.contains(&format!("{METRIC_NAME}{{state=\"{recv}\"}} 111")));
604        assert!(generated.contains(&format!("{METRIC_NAME}{{state=\"{trns}\"}} 444")));
605        assert!(!generated.contains(&format!("{METRIC_NAME}{{state=\"biginvalid\"}} -1")));
606    }
607
608    #[test]
609    fn update_report_drops_stale_states() {
610        let mut registry = Registry::default();
611        let family = register_report(&mut registry);
612        let recv = ReliabilityState::Received.to_string();
613        update_report(&family, HashMap::from([(recv.clone(), 5)]));
614        // e.g. Redis was unavailable
615        update_report(&family, HashMap::new());
616
617        let mut generated = String::new();
618        encode(&mut generated, &registry).unwrap();
619        assert!(!generated.contains(&format!("{METRIC_NAME}{{state=\"{recv}\"}}")));
620    }
621
622    #[test]
623    fn state_ser() {
624        assert_eq!(
625            ReliabilityState::from_str("delivered").unwrap(),
626            ReliabilityState::Delivered
627        );
628        assert_eq!(
629            ReliabilityState::from_str("int_accepted").unwrap(),
630            ReliabilityState::IntAccepted
631        );
632        assert_eq!(
633            serde_json::from_str::<ReliabilityState>(r#""int_accepted""#).unwrap(),
634            ReliabilityState::IntAccepted
635        );
636
637        assert_eq!(ReliabilityState::IntAccepted.to_string(), "int_accepted");
638        assert_eq!(
639            serde_json::to_string(&ReliabilityState::IntAccepted).unwrap(),
640            r#""int_accepted""#
641        );
642    }
643
644    #[actix_rt::test]
645    async fn test_push_reliability_report() -> Result<()> {
646        let mut db = crate::db::mock::MockDbClient::new();
647
648        // Build the state.
649        let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
650        // Remember, we shouldn't just arbitrarily create mid-state values, so
651        // let's start from the beginning
652        let new = ReliabilityState::Received;
653        let old = None;
654        let expr = 1;
655
656        let exp_key = ExpiryKey {
657            id: test_id.clone(),
658            state: new,
659        }
660        .to_string();
661
662        let state_id = format!("state.{test_id}");
663
664        let mut pipeline = redis::Pipeline::new();
665        // We're adding an element so we have something to record.
666        pipeline
667            .cmd("MULTI")
668            // No old state, so no decrement.
669            // "received" is not terminal
670            .cmd("ZADD")
671            .arg(EXPIRY)
672            .arg(expr)
673            .arg(&exp_key)
674            // adjust the counts
675            .cmd("HINCRBY")
676            .arg(COUNTS)
677            .arg(new.to_string())
678            .arg(1)
679            // Run the transaction
680            .cmd("EXEC");
681
682        let mut conn = MockRedisConnection::new(vec![
683            MockCmd::new(
684                redis::cmd("SET")
685                    .arg(&state_id)
686                    .arg(new.to_string())
687                    .arg("NX")
688                    .arg("EX")
689                    .arg(expr),
690                Ok(redis::Value::Okay),
691            ),
692            MockCmd::new(
693                redis::cmd("WATCH").arg(&state_id).arg(EXPIRY),
694                Ok(redis::Value::Okay),
695            ),
696            MockCmd::new(
697                pipeline,
698                Ok(redis::Value::Array(vec![
699                    redis::Value::Okay,                              // MULTI
700                    redis::Value::SimpleString("QUEUED".to_owned()), // ZADD
701                    redis::Value::SimpleString("QUEUED".to_owned()), // HINCRBY
702                    redis::Value::SimpleString("QUEUED".to_owned()), // SET
703                    // Return the transaction results.
704                    redis::Value::Array(vec![
705                        redis::Value::Int(1), // 0 -> 1
706                        redis::Value::Int(1),
707                        // there's no prior value, so this will return Nil
708                        redis::Value::Nil,
709                    ]),
710                ])),
711            ),
712            MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
713        ]);
714
715        let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
716
717        let int_test_id = test_id.clone();
718        db.expect_log_report()
719            .times(1)
720            .withf(move |id, state| id == int_test_id && state == &new)
721            .return_once(|_, _| Ok(()));
722        // test the main report function (note, this does not test redis)
723        let db_box = Box::new(Arc::new(db));
724        let pr = PushReliability::new(
725            &None,
726            db_box.clone(),
727            &metrics,
728            crate::redis_util::MAX_TRANSACTION_LOOP,
729        )
730        .unwrap();
731        // `.record()` uses a pool, so we can't pass the moc connection directly.
732        // Instead, we're going to essentially recreate what `.record()` does calling
733        // the `.internal_record()` function, followed by the database `.record()`.
734        // This emulates
735        // ```
736        // pr.record(&Some(test_id.clone()), new, &None, Some(expr))
737        //    .await?;
738        // ```
739
740        // and mock the redis call.
741        pr.internal_record(&mut conn, &old, new, Some(expr), &test_id)
742            .await?;
743
744        db_box
745            .log_report(&test_id, new)
746            .await
747            .inspect_err(|e| {
748                warn!("πŸ”βš οΈ Unable to record reliability state log: {:?}", e);
749            })
750            .map_err(|e| ApcErrorKind::GeneralError(e.to_string()))?;
751
752        Ok(())
753    }
754
755    #[actix_rt::test]
756    async fn test_push_reliability_record() -> Result<()> {
757        let db = crate::db::mock::MockDbClient::new();
758        let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
759        let new = ReliabilityState::Stored;
760        let old = ReliabilityState::Received;
761        let expr = 1;
762
763        let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
764        let new_key = ExpiryKey {
765            id: test_id.clone(),
766            state: new,
767        }
768        .to_string();
769        let old_key = ExpiryKey {
770            id: test_id.clone(),
771            state: old,
772        }
773        .to_string();
774
775        let state_key = format!("state.{test_id}");
776
777        let mut mock_pipe = redis::Pipeline::new();
778        mock_pipe
779            .cmd("MULTI")
780            .ignore()
781            // Decrement the old state
782            .cmd("HINCRBY")
783            .arg(COUNTS)
784            .arg(old.to_string())
785            .arg(-1)
786            .ignore()
787            // Replace the combined old state key in the expiry set.
788            .cmd("ZREM")
789            .arg(EXPIRY)
790            .arg(old_key)
791            .ignore()
792            .cmd("ZADD")
793            .arg(EXPIRY)
794            .arg(expr)
795            .arg(new_key)
796            .ignore()
797            .cmd("HINCRBY")
798            .arg(COUNTS)
799            .arg(new.to_string())
800            .arg(1)
801            .cmd("SET")
802            .arg(&state_key)
803            .arg(new.to_string())
804            .arg("XX")
805            .arg("GET")
806            .arg("KEEPTTL")
807            .ignore()
808            .cmd("EXEC")
809            .ignore();
810
811        let mut conn = MockRedisConnection::new(vec![
812            // Create the new state
813            MockCmd::new(
814                redis::cmd("WATCH").arg(&state_key).arg(EXPIRY),
815                Ok(redis::Value::Okay),
816            ),
817            MockCmd::new(
818                redis::cmd("GET").arg(&state_key),
819                Ok(redis::Value::SimpleString(old.to_string())),
820            ),
821            // NOTE: Technically, since we `.ignore()` these, we could just have a
822            // Vec containing just `Okay`. I'm being a bit pedantic here because I know
823            // that this will come back to haunt me if I'm not, and because figuring out
824            // the proper response for this was annoying.
825            MockCmd::new(
826                mock_pipe,
827                Ok(redis::Value::Array(vec![
828                    redis::Value::Okay,
829                    // Match the number of commands that are being held for processing
830                    redis::Value::SimpleString("QUEUED".to_owned()), // multi
831                    redis::Value::SimpleString("QUEUED".to_owned()), // hincrby
832                    redis::Value::SimpleString("QUEUED".to_owned()), // zrem
833                    redis::Value::SimpleString("QUEUED".to_owned()), // zadd
834                    redis::Value::SimpleString("QUEUED".to_owned()), // hincrby
835                    redis::Value::SimpleString("QUEUED".to_owned()), // set
836                    // the exec has been called, return an array containing the results.
837                    redis::Value::Array(vec![
838                        redis::Value::Okay, //Multi
839                        redis::Value::Okay, //hincrby
840                        redis::Value::Okay, //zrem
841                        redis::Value::Okay, //zadd
842                        redis::Value::Okay, //hincr
843                        redis::Value::SimpleString(old.to_string()),
844                    ]),
845                ])),
846            ),
847            // If the transaction fails, this should return a redis::Value::Nil
848            MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
849        ]);
850
851        // test the main report function (note, this does not test redis)
852        let pr = PushReliability::new(
853            &None,
854            Box::new(Arc::new(db)),
855            &metrics,
856            crate::redis_util::MAX_TRANSACTION_LOOP,
857        )
858        .unwrap();
859        let _ = pr
860            .internal_record(&mut conn, &Some(old), new, Some(expr), &test_id)
861            .await;
862
863        Ok(())
864    }
865
866    #[actix_rt::test]
867    async fn test_push_reliability_full() -> Result<()> {
868        let db = crate::db::mock::MockDbClient::new();
869        let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
870        let new = ReliabilityState::Received;
871        let stored = ReliabilityState::Stored;
872        let expr = 1;
873
874        let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
875        let new_key = ExpiryKey {
876            id: test_id.clone(),
877            state: new,
878        }
879        .to_string();
880        let stored_key = ExpiryKey {
881            id: test_id.clone(),
882            state: stored,
883        }
884        .to_string();
885
886        let state_key = format!("state.{test_id}");
887
888        let mut mock_pipe = redis::Pipeline::new();
889        mock_pipe
890            .cmd("MULTI")
891            .ignore()
892            // Decrement the old state count
893            .cmd("HINCRBY")
894            .arg(COUNTS)
895            .arg(stored.to_string())
896            .arg(-1)
897            .ignore()
898            // Replace the expiry key
899            .cmd("ZREM")
900            .arg(EXPIRY)
901            .arg(stored_key.to_string())
902            .ignore()
903            .cmd("ZADD")
904            .arg(EXPIRY)
905            .arg(expr)
906            .arg(new_key)
907            .ignore()
908            // Increment the new state count
909            .cmd("HINCRBY")
910            .arg(COUNTS)
911            .arg(new.to_string())
912            .arg(1)
913            .ignore()
914            // And create the new state transition key (since the message is "live" again.)
915            .cmd("SET")
916            .arg(&state_key)
917            .arg(new.to_string())
918            .arg("XX")
919            .arg("GET")
920            .cmd("EXEC")
921            .ignore();
922
923        let mut conn = MockRedisConnection::new(vec![
924            MockCmd::new(
925                // Create the new state
926                redis::cmd("SET")
927                    .arg(format!("state.{test_id}"))
928                    .arg(new.to_string())
929                    .arg("NX")
930                    .arg("EX")
931                    .arg(expr),
932                Ok(redis::Value::Okay),
933            ),
934            // increment the count for new
935            MockCmd::new(
936                redis::cmd("HINCRBY")
937                    .arg(COUNTS)
938                    .arg(new.to_string())
939                    .arg(1),
940                Ok(redis::Value::Okay),
941            ),
942            // begin the transaction
943            MockCmd::new(
944                redis::cmd("WATCH").arg(&state_key).arg(EXPIRY),
945                Ok(redis::Value::Okay),
946            ),
947            // NOTE: Technically, since we `.ignore()` these, we could just have a
948            // Vec containing just `Okay`. I'm being a bit pedantic here because I know
949            // that this will come back to haunt me if I'm not, and because figuring out
950            // the proper response for this was annoying.
951            MockCmd::new(
952                mock_pipe,
953                Ok(redis::Value::Array(vec![
954                    redis::Value::Okay,
955                    // Match the number of commands that are being held for processing
956                    redis::Value::SimpleString("QUEUED".to_owned()),
957                    redis::Value::SimpleString("QUEUED".to_owned()),
958                    redis::Value::SimpleString("QUEUED".to_owned()),
959                    redis::Value::SimpleString("QUEUED".to_owned()),
960                    // the exec has been called, return an array containing the results.
961                    redis::Value::Array(vec![
962                        redis::Value::Okay,
963                        redis::Value::Okay,
964                        redis::Value::Okay,
965                        redis::Value::Okay,
966                    ]),
967                ])),
968            ),
969            // If the transaction fails, this should return a redis::Value::Nil
970            MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
971        ]);
972
973        // test the main report function (note, this does not test redis)
974        let pr = PushReliability::new(
975            &None,
976            Box::new(Arc::new(db)),
977            &metrics,
978            crate::redis_util::MAX_TRANSACTION_LOOP,
979        )
980        .unwrap();
981        let _ = pr
982            .internal_record(&mut conn, &Some(stored), new, Some(expr), &test_id)
983            .await;
984
985        Ok(())
986    }
987
988    #[actix_rt::test]
989    async fn test_push_reliability_gc() -> Result<()> {
990        let db = crate::db::mock::MockDbClient::new();
991        let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
992        let new = ReliabilityState::Accepted;
993        let key = format!("{}#{}", &test_id, &new);
994        let expr = 1;
995        let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
996
997        let response: redis::Value = redis::Value::Array(vec![redis::Value::SimpleString(
998            ExpiryKey {
999                id: test_id.clone(),
1000                state: new,
1001            }
1002            .to_string(),
1003        )]);
1004
1005        // Construct the Pipeline.
1006        // A redis "pipeline" is a set of instructions that are executed in order. These
1007        // are not truly atomic, as other actions can interrupt a pipeline, but they
1008        // are guaranteed to happen in sequence. Transactions essentially check that the
1009        // WATCH key is not altered before the pipeline is executed.
1010        let mut mock_pipe = redis::Pipeline::new();
1011        mock_pipe
1012            .cmd("MULTI")
1013            .ignore()
1014            // Adjust the state counts.
1015            .cmd("HINCRBY")
1016            .arg(COUNTS)
1017            .arg(new.to_string())
1018            .arg(-1)
1019            .ignore()
1020            .cmd("HINCRBY")
1021            .arg(COUNTS)
1022            .arg(ReliabilityState::Expired.to_string())
1023            .arg(1)
1024            // Replace the state key in the expiry set.
1025            .ignore()
1026            .cmd("ZREM")
1027            .arg(EXPIRY)
1028            .arg(key)
1029            .ignore()
1030            .cmd("EXEC")
1031            .ignore();
1032
1033        // This mocks the "conn"ection, so all commands sent to the connection need
1034        // to be separated out. This includes the transaction "WATCH" and "UNWATCH".
1035        // The "meat" of the functions are handled via a Pipeline, which conjoins the
1036        // commands into one group (see above). Pipelines return an array of
1037        // results, one entry for each pipeline command.
1038        // In other words, the `internal_gc` commands are probably in the pipeline,
1039        // all the others are probably in the conn.
1040        let mut conn = MockRedisConnection::new(vec![
1041            MockCmd::new(redis::cmd("WATCH").arg(EXPIRY), Ok(redis::Value::Okay)),
1042            MockCmd::new(
1043                redis::cmd("ZRANGEBYSCORE").arg(EXPIRY).arg(0).arg(expr),
1044                Ok(response),
1045            ),
1046            // NOTE: Technically, since we `.ignore()` these, we could just have a
1047            // Vec containing just `Okay`. I'm being a bit pedantic here because I know
1048            // that this will come back to haunt me if I'm not, and because figuring out
1049            // the proper response for this was annoying.
1050            MockCmd::new(
1051                mock_pipe,
1052                Ok(redis::Value::Array(vec![
1053                    redis::Value::Okay,
1054                    // Match the number of commands that are being held for processing
1055                    redis::Value::SimpleString("QUEUED".to_owned()),
1056                    redis::Value::SimpleString("QUEUED".to_owned()),
1057                    redis::Value::SimpleString("QUEUED".to_owned()),
1058                    // the exec has been called, return an array containing the results.
1059                    redis::Value::Array(vec![
1060                        redis::Value::Okay,
1061                        redis::Value::Okay,
1062                        redis::Value::Okay,
1063                    ]),
1064                ])),
1065            ),
1066            // If the transaction fails, this should return a redis::Value::Nil
1067            MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
1068        ]);
1069
1070        // test the main report function (note, this does not test redis)
1071        let pr = PushReliability::new(
1072            &None,
1073            Box::new(Arc::new(db)),
1074            &metrics,
1075            crate::redis_util::MAX_TRANSACTION_LOOP,
1076        )
1077        .unwrap();
1078        // functionally a no-op, but it does exercise lines.
1079        pr.gc().await?;
1080
1081        // and mock the redis call.
1082        pr.internal_gc(&mut conn, 1).await?;
1083
1084        Ok(())
1085    }
1086}