autopush_common/reliability.rs
1/// Push Reliability Recorder
2///
3/// This allows us to track messages from select, known parties (currently, just
4/// mozilla generated and consumed) so that we can identify potential trouble spots
5/// and where messages expire early. Message expiration can lead to message loss
6use std::collections::HashMap;
7use std::str::FromStr;
8use std::sync::Arc;
9
10use cadence::StatsdClient;
11use chrono::TimeDelta;
12use deadpool_redis::Config;
13
14use prometheus_client::{metrics::family::Family, metrics::gauge::Gauge, registry::Registry};
15use redis::{AsyncCommands, aio::ConnectionLike};
16use redis::{Pipeline, Value};
17
18use crate::db::client::DbClient;
19use crate::errors::{ApcError, ApcErrorKind, Result};
20use crate::metric_name::MetricName;
21use crate::metrics::StatsdClientExt;
22use crate::util::timing::sec_since_epoch;
23
24// Redis Keys
25pub const COUNTS: &str = "state_counts";
26pub const EXPIRY: &str = "expiry";
27
28const CONNECTION_EXPIRATION: TimeDelta = TimeDelta::seconds(10);
29// Minimum expiration period of 1 second.
30// This was set to `0`, but there was some confusion whether that would not set an
31// expiration time for a record or would set a record not to expire.
32const MIN_EXPIRATION: u64 = 1;
33
34/// The various states that a message may transit on the way from reception to delivery.
35// Note: "Message" in this context refers to the Subscription Update.
36// TODO: Differentiate between "transmitted via webpush" and "transmitted via bridge"?
37#[derive(
38 Debug,
39 Clone,
40 Copy,
41 PartialEq,
42 Eq,
43 serde::Deserialize,
44 serde::Serialize,
45 strum::Display,
46 strum::EnumString,
47)]
48#[serde(rename_all = "snake_case")]
49#[strum(serialize_all = "snake_case")]
50pub enum ReliabilityState {
51 Received, // Message was received by the Push Server
52 Stored, // Message was stored because it could not be delivered immediately
53 Retrieved, // Message was taken from storage for delivery
54 IntTransmitted, // Message was handed off between autoendpoint and autoconnect
55 IntAccepted, // Message was accepted by autoconnect from autopendpoint
56 BridgeTransmitted, // Message was handed off to a mobile bridge for eventual delivery
57 Transmitted, // Message was handed off for delivery to the UA
58 Accepted, // Message was accepted for delivery by the UA
59 Delivered, // Message was provided to the WebApp recipient by the UA
60 DecryptionError, // Message was provided to the UA and it reported a decryption error
61 NotDelivered, // Message was provided to the UA and it reported a not delivered error
62 Expired, // Message expired naturally (e.g. TTL=0)
63 Errored, // Message resulted in an Error state and failed to be delivered.
64}
65
66impl ReliabilityState {
67 /// Has the message reached a state where no further transitions should be possible?
68 pub fn is_terminal(self) -> bool {
69 // NOTE: this list should match the daily snapshot captured by `reliability_cron.py`
70 // which will trim these counts after the max message TTL has expired.
71 matches!(
72 self,
73 ReliabilityState::DecryptionError
74 | ReliabilityState::BridgeTransmitted
75 | ReliabilityState::Delivered
76 | ReliabilityState::Errored
77 | ReliabilityState::Expired
78 | ReliabilityState::NotDelivered
79 )
80 }
81}
82
83#[derive(Clone)]
84pub struct PushReliability {
85 pool: Option<deadpool_redis::Pool>,
86 db: Box<dyn DbClient>,
87 metrics: Arc<StatsdClient>,
88 retries: usize,
89}
90
91// Define a struct to hold the expiry key, since it's easy to flub the order.
92pub struct ExpiryKey {
93 pub id: String,
94 pub state: ReliabilityState,
95}
96
97impl std::fmt::Display for ExpiryKey {
98 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
99 write!(f, "{}#{}", self.id, self.state)
100 }
101}
102
103impl TryFrom<String> for ExpiryKey {
104 type Error = ApcError;
105 fn try_from(value: String) -> Result<Self> {
106 let (id, state) = value.split_once('#').ok_or_else(|| {
107 ApcErrorKind::GeneralError("ExpiryKey must be in the format 'id#state'".to_owned())
108 })?;
109 let state: ReliabilityState = ReliabilityState::from_str(state).map_err(|_| {
110 ApcErrorKind::GeneralError(
111 "Invalid state in ExpiryKey, must be a valid ReliabilityState".to_owned(),
112 )
113 })?;
114 Ok(Self {
115 id: id.to_owned(),
116 state,
117 })
118 }
119}
120
121impl PushReliability {
122 // Do the magic to make a report instance, whatever that will be.
123 pub fn new(
124 reliability_dsn: &Option<String>,
125 db: Box<dyn DbClient>,
126 metrics: &Arc<StatsdClient>,
127 retries: usize,
128 ) -> Result<Self> {
129 let Some(reliability_dsn) = reliability_dsn else {
130 debug!("π No reliability DSN declared.");
131 return Ok(Self {
132 pool: None,
133 db: db.clone(),
134 metrics: metrics.clone(),
135 retries,
136 });
137 };
138
139 let config: deadpool_redis::Config = Config::from_url(reliability_dsn);
140 let pool = Some(
141 config
142 .builder()
143 .map_err(|e| {
144 ApcErrorKind::GeneralError(format!("Could not config reliability pool {e:?}"))
145 })?
146 .create_timeout(Some(CONNECTION_EXPIRATION.to_std().unwrap()))
147 .runtime(deadpool::Runtime::Tokio1)
148 .build()
149 .map_err(|e| {
150 ApcErrorKind::GeneralError(format!("Could not build reliability pool {e:?}"))
151 })?,
152 );
153
154 Ok(Self {
155 pool,
156 db: db.clone(),
157 metrics: metrics.clone(),
158 retries,
159 })
160 }
161
162 // Record the message state change to storage.
163 pub async fn record(
164 &self,
165 reliability_id: &Option<String>,
166 new: ReliabilityState,
167 old: &Option<ReliabilityState>,
168 expr: Option<u64>,
169 ) -> Result<Option<ReliabilityState>> {
170 let Some(id) = reliability_id else {
171 return Ok(None);
172 };
173 if let Some(pool) = &self.pool {
174 debug!(
175 "π {} from {} to {}",
176 id,
177 old.map(|v| v.to_string())
178 .unwrap_or_else(|| "None".to_owned()),
179 new
180 );
181 match pool.get().await {
182 Ok(mut conn) => {
183 self.internal_record(&mut conn, old, new, expr, id)
184 .await
185 .map_err(|e| {
186 warn!("πβ οΈ Unable to record reliability state: {:?}", e);
187 ApcErrorKind::GeneralError(
188 "Could not record reliability state".to_owned(),
189 )
190 })?;
191 }
192 Err(e) => warn!("πβ οΈ Unable to get reliability state pool, {:?}", e),
193 };
194 };
195 // Errors are not fatal, and should not impact message flow, but
196 // we should record them somewhere.
197 let _ = self.db.log_report(id, new).await.inspect_err(|e| {
198 warn!("πβ οΈ Unable to record reliability state log: {:?}", e);
199 });
200 Ok(Some(new))
201 }
202
203 /// Record the state change in the reliability datastore.
204 /// Because of a strange duplication error, we're using three "tables".
205 /// The COUNTS table contains the count of messages in each state. It's provided as a quick lookup
206 /// for the dashboard query.
207 /// The EXPIRY table contains the expiration time for messages. Those are cleaned up by the
208 /// `gc` function.
209 /// Finally, there's the "table" that is `state.{id}`. This contains the current state of each
210 /// message. Ideally, this would use something like `HSETEX`, but that is not available until
211 /// Redis 8.0, so for now, we use the `SET` function, which provides an expiration time.
212 /// BE SURE TO SET AN EXPIRATION TIME FOR EACH STATE MESSAGE!
213 // TODO: Do we need to keep the state in the message? Probably good for sanity reasons.
214 pub(crate) async fn internal_record<C: ConnectionLike + AsyncCommands>(
215 &self,
216 conn: &mut C,
217 old: &Option<ReliabilityState>,
218 new: ReliabilityState,
219 expr: Option<u64>,
220 id: &str,
221 ) -> Result<()> {
222 trace!(
223 "π internal record: {} from {} to {}",
224 id,
225 old.map(|v| v.to_string())
226 .unwrap_or_else(|| "None".to_owned()),
227 new
228 );
229
230 let state_key = format!("state.{id}");
231 trace!("π state key: {}", &state_key);
232 /*
233 ## Reliability lock and adjustment
234
235 This transaction creates a lock on the individual messages "state_key" (which contains the current messages state)
236 and the entire Expiration table. If either of those values are changed while the transaction is in progress, then
237 the transaction will fail, return a `Redis::Nil`, and the transaction will retry up to `self.retries` (Note:
238 none of the operations in the transaction will have taken place yet.)
239
240 We want to lock on the `state_key` because that shows the given state of the message.
241 We want to lock on the `expiry` table because the message may have expired, been handled by the `gc()` function and
242 may have already been adjusted. (Remember, the `expiry` table holds timestamp markers for when a message will be
243 expiring.)
244
245 The operations in the Pipeline are:
246
247 1. If there is an `old` state, decrement the old state count and remove the old marker from the `expiry` table.
248 2. If the `new` state is not a terminal state (e.g. it is not a final disposition for a message), then create a new
249 entry for `expiry`
250 3. Increment the `new` state count
251 4. Modify the `state.{ID}` value (if it exists) to indicate the newest known state for the message, returning the
252 prior value. Checking that this value is not Nil is an additional sanity check that the value was not removed
253 or altered by a different process. (We are already imposing a transaction lock which should prevent this, but
254 additional paranoia can sometimes be good.)
255
256 Upon execution of the Pipeline, we get back a list of results. This should look similar to a `Queued` entry for
257 every function that we've performed, followed by an inline list of the results of those functions. (e.g.
258 for a transition between `ReliabilityState::Stored` to `ReliabilityState::Retrieved`, which are neither terminal
259 states), we should see something like: `["Queued","Queued","Queued",["OK", "OK", "stored"]]`
260
261 If any of the locks failed, we should get back a `Nil`, in which case, we would want to try this operation again.
262 In addition, the sanity `Get` may also return a `Nil` which would also indicate that there was a problem. There's
263 some debate whether or not to retry the operation if that's the case (since it would imply that the lock failed
264 for some unexpected reason), however for now, we just report a soft `error!()`.
265
266 */
267 // Create the initial state key.
268 // Do not place this inside of the transaction. We monitor the state key and the transaction will
269 // fail because the value will change before the transaction completes. Yes, really.
270 if new == ReliabilityState::Received && old.is_none() {
271 trace!(
272 "π Creating new record {state_key} ex {:?}",
273 expr.unwrap_or(MIN_EXPIRATION)
274 );
275 // we can't perform this in a transaction because we can only increment if the set succeeds,
276 // and values aren't returned when creating values in transactions. In order to do this
277 // from inside the transaction, you would need to create a function, and that feels a bit
278 // too heavy for this.
279 // Create the new `state.{id}` key if it does not exist, and set the expiration.
280 let options = redis::SetOptions::default()
281 .with_expiration(redis::SetExpiry::EX(expr.unwrap_or(MIN_EXPIRATION)))
282 .conditional_set(redis::ExistenceCheck::NX);
283 trace!("π β SET {state_key} NX EX {:?}", new);
284 let result = conn
285 .set_options::<_, _, Value>(&state_key, new.to_string(), options)
286 .await
287 .map_err(|e| {
288 //dbg!(&e);
289 warn!("πβ οΈ Could not create state key: {:?}", e);
290 ApcErrorKind::GeneralError("Could not create the state key".to_owned())
291 })?;
292 if result != redis::Value::Okay {
293 // Redis returned a `Nil`, indicating that there was some error. The only thing that should cause that
294 // would be if the `old` state was reset to `None` and we thought we needed to create a new state.
295 // Since the message carries it's prior state, it shouldn't be set to `None` unless there's something
296 // strange going on.
297 // TODO: It's worth noting that when restarting autoendpoint, we get a large number of these in the logs.
298 // Need to figure out the reason for that.
299 // The `result` is always `nil` so that's not helpful.
300 error!("πβ οΈ Tried to recreate state_key {state_key}: {old:?} => {new:?}",);
301 return Err(
302 ApcErrorKind::GeneralError("Tried to recreate state_key".to_string()).into(),
303 );
304 }
305 } else {
306 trace!("π Checking {:?}", &old);
307 // safety check (yes, there's still a slight chance of a race, but it's small)
308 if let Some(old) = old {
309 let check_state: String = conn.get(&state_key).await?;
310 trace!("π Checking state for {}: {:?}", id, &check_state);
311 if check_state != old.to_string() {
312 trace!(
313 "π Attempting to update state for {} from {} to {}, but current state is different: {:?}",
314 id, old, new, check_state
315 );
316 return Err(ApcErrorKind::GeneralError(
317 "State mismatch during reliability record update".to_owned(),
318 )
319 .into());
320 }
321 };
322 }
323
324 crate::redis_util::transaction(
325 conn,
326 &[&state_key, &EXPIRY.to_owned()],
327 self.retries,
328 || ApcErrorKind::GeneralError("Exceeded reliability record retry attempts".to_owned()),
329 async |conn, pipe: &mut Pipeline| {
330 // The first state is special, since we need to create the `state_key`.
331 // remove the old state from the expiry set, if it exists.
332 // There should only be one message at a given state in the `expiry` table.
333 // Since we only use that table to track messages that may expire. (We
334 // decrement "expired" messages in the `gc` function, so having messages
335 // in multiple states may decrement counts incorrectly.))
336 if let Some(old) = old {
337 trace!("π β {old} - {id}");
338 trace!("π πͺβ HINCRBY {COUNTS} 1");
339 pipe.hincr(COUNTS, old.to_string(), -1);
340 let key = ExpiryKey {
341 id: id.to_string(),
342 state: old.to_owned(),
343 }
344 .to_string();
345 trace!("π internal remove old state: {:?}", key);
346 trace!("π πͺβ ZREM {EXPIRY} {key}");
347 pipe.zrem(EXPIRY, &key);
348 }
349 if !new.is_terminal() {
350 // Write the expiration only if the state is non-terminal. Otherwise we run the risk of
351 // messages reporting a false "expired" state even if they were "successful".
352 let key = ExpiryKey {
353 id: id.to_string(),
354 state: new.to_owned(),
355 }
356 .to_string();
357 trace!("π πͺβ ZADD {EXPIRY} {:?} {key}", expr.unwrap_or_default());
358 pipe.zadd(EXPIRY, &key, expr.unwrap_or_default());
359 trace!("π internal record result: {:?}", key);
360 }
361 trace!("π β {new} - {id}");
362 // Bump up the new state count, and set the state key's state if it still exists.
363 trace!("π πͺβ HINCRBY {COUNTS} {new} 1");
364 pipe.hincr(COUNTS, new.to_string(), 1);
365 if new != ReliabilityState::Received {
366 let options = redis::SetOptions::default()
367 .with_expiration(redis::SetExpiry::KEEPTTL)
368 .conditional_set(redis::ExistenceCheck::XX)
369 .get(true);
370 trace!("π πͺβ SET {state_key} {new} XX KEEPTTL");
371 pipe.set_options(&state_key, new.to_string(), options);
372 }
373 // `exec_query` returns `RedisResult<()>`.
374 // `query_async` returns `RedisResult<Option<redis::Value>, RedisError>`.
375 // We really don't care about the returned result here, but transaction
376 // retries if we return Ok(None), so we run the exec and return
377 // a nonce `Some` value.
378 // The turbo-fish is a fallback for edition 2024
379 let result = pipe.query_async::<redis::Value>(conn).await?;
380 trace!("π πͺ {id} - {:?}", &result);
381 // The last element returned from the command is the result of the pipeline.
382 // If Redis encounters an error, it will return a `nil` as well. We handle both
383 // the same (retry), so we can normalize errors as `nil`.
384 // The last of which should be the result of the piped command set.
385 // This should return `nil` if there is any error, in which case we record
386 // a soft error. (On success, it will return the result of the last command
387 // in the pipe, which may vary due to the current state).
388 // This could also be strung together as a cascade of functions, but it's broken
389 // out to discrete steps for readability.
390 /* On prod, we get a large number of these errors, which I think might be clogging
391 up the redis connections, causing servers to report as degraded.
392 */
393 if result == redis::Value::Nil {
394 warn!("πβ πͺ {id} - Pipe failed, skipping retry.");
395 // temporarily just let things fail to handle autoendpoint degradation.
396 // return Ok(None);
397 return Ok(Some(redis::Value::Okay));
398 }
399 if let Some(operations) = result.as_sequence() {
400 // We have responses, the first items report the state of the commands,
401 // the final line is a list of command results.
402 if let Some(result_values) = operations.last() && let Some(results) = result_values.as_sequence() &&
403 // The last command should contain the prior state. If it returned `Nil`
404 // for some, unexpected reason, note the error.
405 Some(&redis::Value::Nil) == results.last() {
406 error!("ππ¨ WARNING: Lock Issue for {id}")
407 // There is some debate about whether or not to rerun
408 // the transaction if this state is reached.
409 // Rerunning would cause the counts to be impacted, but
410 // might address any other issues that caused the
411 // `Nil` to be returned.
412 // For now, let's just log the error.
413
414
415 }
416 Ok(Some(redis::Value::Okay))
417 } else {
418 // a Nil will rerun the transaction.
419 Ok(None)
420 }
421 },
422 )
423 .await
424 .inspect_err(|e| {
425 //dbg!(&e);
426 warn!("πβ οΈError occurred during transaction: {:?}", e);
427 })?;
428 Ok(())
429 }
430
431 /// Perform a garbage collection cycle on a reliability object.
432 pub async fn gc(&self) -> Result<()> {
433 if let Some(pool) = &self.pool
434 && let Ok(mut conn) = pool.get().await
435 {
436 debug!("π performing pre-report garbage collection");
437 return self.internal_gc(&mut conn, sec_since_epoch()).await;
438 }
439 Ok(())
440 }
441
442 // Perform the `garbage collection` cycle. This will scan the currently known timestamp
443 // indexed entries in redis looking for "expired" data, and then rectify the counts to
444 // indicate the final states. This is because many of the storage systems do not provide
445 // indicators when data reaches a TTL.
446
447 // A few notes about redis:
448 // "pipeline" essentially stitches commands together. Each command executes in turn, but the data store
449 // remains open for other operations to occur.
450 // `atomic()` wraps things in a transaction, essentially locking the data store while the command executes.
451 // Sadly, there's no way to create a true transaction where you read and write in a single operation, so we
452 // have to presume some "slop" here.
453 pub(crate) async fn internal_gc<C: ConnectionLike + AsyncCommands>(
454 &self,
455 conn: &mut C,
456 expr: u64,
457 ) -> Result<()> {
458 let result: redis::Value = crate::redis_util::transaction(
459 conn,
460 &[EXPIRY],
461 self.retries,
462 || ApcErrorKind::GeneralError("Exceeded gc retry attempts".to_owned()),
463 async |conn, pipe| {
464 // First, get the list of values that are to be purged.
465 let purged: Vec<String> = conn.zrangebyscore(EXPIRY, 0, expr as isize).await?;
466 // insta-bail if there's nothing to do.
467 if purged.is_empty() {
468 return Ok(Some(redis::Value::Nil));
469 }
470
471 // Now purge each of the values by resetting the counts and removing the item from
472 // the purge set.
473 for key in purged {
474 let Ok(expiry_key) = ExpiryKey::try_from(key.clone()) else {
475 let err = "Invalid key stored in Reliability datastore";
476 error!("ππ₯ {} [{:?}]", &err, &key);
477 return Err(ApcErrorKind::GeneralError(err.to_owned()));
478 };
479 // Adjust the COUNTS and then remove the record from the list of expired rows.
480 trace!("π βπͺ¦ {} - {key}", expiry_key.state.to_string());
481 pipe.hincr(COUNTS, expiry_key.state.to_string(), -1);
482 pipe.hincr(COUNTS, ReliabilityState::Expired.to_string(), 1);
483 pipe.zrem(EXPIRY, key);
484 }
485 Ok(pipe.query_async(conn).await?)
486 },
487 )
488 .await?;
489 self.metrics
490 .incr_with_tags(MetricName::ReliabilityGc)
491 .with_tag(
492 "status",
493 if result == redis::Value::Nil {
494 "error"
495 } else {
496 "success"
497 },
498 )
499 .send();
500 Ok(())
501 }
502
503 // Return a snapshot of milestone states
504 // This will probably not be called directly, but useful for debugging.
505 pub async fn report(&self) -> Result<HashMap<String, i32>> {
506 if let Some(pool) = &self.pool
507 && let Ok(mut conn) = pool.get().await
508 {
509 return Ok(conn.hgetall(COUNTS).await.map_err(|e| {
510 ApcErrorKind::GeneralError(format!("Could not read report {e:?}"))
511 })?);
512 }
513 Ok(HashMap::new())
514 }
515
516 pub async fn health_check<'a>(&self) -> Result<&'a str> {
517 if let Some(pool) = &self.pool {
518 let mut conn = pool.get().await.map_err(|e| {
519 ApcErrorKind::GeneralError(format!(
520 "Could not connect to reliability datastore: {e:?}"
521 ))
522 })?;
523 // Add a type here, even though we're tossing the value, in order to prevent the `FromRedisValue` warning.
524 conn.ping::<()>().await.map_err(|e| {
525 ApcErrorKind::GeneralError(format!("Could not ping reliability datastore: {e:?}"))
526 })?;
527 Ok("OK")
528 } else {
529 Ok("OK")
530 }
531 }
532}
533
534const METRIC_NAME: &str = "autopush_reliability";
535
536/// A "family" is a grouping of metrics.
537/// we specify this as the ("label", "label value") which index to a Gauge.
538pub type ReportFamily = Family<Vec<(&'static str, String)>, Gauge>;
539
540/// Register the reliability state gauges with the given registry.
541pub fn register_report(registry: &mut Registry) -> ReportFamily {
542 let family = ReportFamily::default();
543 // This creates the top level association of the elements in the family with the metric.
544 registry.register(
545 METRIC_NAME,
546 "Count of messages at given states",
547 family.clone(),
548 );
549 family
550}
551
552/// Set the reliability state gauges from the given milestone counts.
553///
554/// The family is long lived, so it's cleared first: states missing from
555/// `values` (e.g. when Redis is unavailable) are dropped rather than left at
556/// their last known value.
557pub fn update_report(family: &ReportFamily, values: HashMap<String, i32>) {
558 family.clear();
559 for (milestone, value) in values.into_iter() {
560 // prevent any stray leakage of invalid state data
561 if ReliabilityState::from_str(&milestone).is_err() {
562 trace!("π skipping invalid state {milestone:?}");
563 continue;
564 }
565 // Specify the static "state" label name with the given milestone, and add the
566 // value as the gauge value.
567 family
568 .get_or_create(&vec![("state", milestone)])
569 .set(value.into());
570 }
571}
572
573#[cfg(test)]
574mod tests {
575 use std::collections::HashMap;
576 use std::str::FromStr;
577
578 use super::*;
579 use prometheus_client::encoding::text::encode;
580 use redis_test::{MockCmd, MockRedisConnection};
581 use uuid::Uuid;
582
583 #[test]
584 fn test_report() {
585 // create a nonce report
586 let mut report: HashMap<String, i32> = HashMap::new();
587 let recv = ReliabilityState::Received.to_string();
588 let trns = ReliabilityState::Transmitted.to_string();
589 report.insert(recv.clone(), 111);
590 report.insert(ReliabilityState::Stored.to_string(), 222);
591 report.insert(ReliabilityState::Retrieved.to_string(), 333);
592 report.insert(trns.clone(), 444);
593 report.insert("biginvalid".to_string(), -1);
594
595 let mut registry = Registry::default();
596 let family = register_report(&mut registry);
597 update_report(&family, report);
598 let mut generated = String::new();
599 encode(&mut generated, ®istry).unwrap();
600 // We don't really care if the `Created` or `HELP` lines are included
601 assert!(generated.contains(&format!("# TYPE {METRIC_NAME}")));
602 // sample the first and last values.
603 assert!(generated.contains(&format!("{METRIC_NAME}{{state=\"{recv}\"}} 111")));
604 assert!(generated.contains(&format!("{METRIC_NAME}{{state=\"{trns}\"}} 444")));
605 assert!(!generated.contains(&format!("{METRIC_NAME}{{state=\"biginvalid\"}} -1")));
606 }
607
608 #[test]
609 fn update_report_drops_stale_states() {
610 let mut registry = Registry::default();
611 let family = register_report(&mut registry);
612 let recv = ReliabilityState::Received.to_string();
613 update_report(&family, HashMap::from([(recv.clone(), 5)]));
614 // e.g. Redis was unavailable
615 update_report(&family, HashMap::new());
616
617 let mut generated = String::new();
618 encode(&mut generated, ®istry).unwrap();
619 assert!(!generated.contains(&format!("{METRIC_NAME}{{state=\"{recv}\"}}")));
620 }
621
622 #[test]
623 fn state_ser() {
624 assert_eq!(
625 ReliabilityState::from_str("delivered").unwrap(),
626 ReliabilityState::Delivered
627 );
628 assert_eq!(
629 ReliabilityState::from_str("int_accepted").unwrap(),
630 ReliabilityState::IntAccepted
631 );
632 assert_eq!(
633 serde_json::from_str::<ReliabilityState>(r#""int_accepted""#).unwrap(),
634 ReliabilityState::IntAccepted
635 );
636
637 assert_eq!(ReliabilityState::IntAccepted.to_string(), "int_accepted");
638 assert_eq!(
639 serde_json::to_string(&ReliabilityState::IntAccepted).unwrap(),
640 r#""int_accepted""#
641 );
642 }
643
644 #[actix_rt::test]
645 async fn test_push_reliability_report() -> Result<()> {
646 let mut db = crate::db::mock::MockDbClient::new();
647
648 // Build the state.
649 let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
650 // Remember, we shouldn't just arbitrarily create mid-state values, so
651 // let's start from the beginning
652 let new = ReliabilityState::Received;
653 let old = None;
654 let expr = 1;
655
656 let exp_key = ExpiryKey {
657 id: test_id.clone(),
658 state: new,
659 }
660 .to_string();
661
662 let state_id = format!("state.{test_id}");
663
664 let mut pipeline = redis::Pipeline::new();
665 // We're adding an element so we have something to record.
666 pipeline
667 .cmd("MULTI")
668 // No old state, so no decrement.
669 // "received" is not terminal
670 .cmd("ZADD")
671 .arg(EXPIRY)
672 .arg(expr)
673 .arg(&exp_key)
674 // adjust the counts
675 .cmd("HINCRBY")
676 .arg(COUNTS)
677 .arg(new.to_string())
678 .arg(1)
679 // Run the transaction
680 .cmd("EXEC");
681
682 let mut conn = MockRedisConnection::new(vec![
683 MockCmd::new(
684 redis::cmd("SET")
685 .arg(&state_id)
686 .arg(new.to_string())
687 .arg("NX")
688 .arg("EX")
689 .arg(expr),
690 Ok(redis::Value::Okay),
691 ),
692 MockCmd::new(
693 redis::cmd("WATCH").arg(&state_id).arg(EXPIRY),
694 Ok(redis::Value::Okay),
695 ),
696 MockCmd::new(
697 pipeline,
698 Ok(redis::Value::Array(vec![
699 redis::Value::Okay, // MULTI
700 redis::Value::SimpleString("QUEUED".to_owned()), // ZADD
701 redis::Value::SimpleString("QUEUED".to_owned()), // HINCRBY
702 redis::Value::SimpleString("QUEUED".to_owned()), // SET
703 // Return the transaction results.
704 redis::Value::Array(vec![
705 redis::Value::Int(1), // 0 -> 1
706 redis::Value::Int(1),
707 // there's no prior value, so this will return Nil
708 redis::Value::Nil,
709 ]),
710 ])),
711 ),
712 MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
713 ]);
714
715 let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
716
717 let int_test_id = test_id.clone();
718 db.expect_log_report()
719 .times(1)
720 .withf(move |id, state| id == int_test_id && state == &new)
721 .return_once(|_, _| Ok(()));
722 // test the main report function (note, this does not test redis)
723 let db_box = Box::new(Arc::new(db));
724 let pr = PushReliability::new(
725 &None,
726 db_box.clone(),
727 &metrics,
728 crate::redis_util::MAX_TRANSACTION_LOOP,
729 )
730 .unwrap();
731 // `.record()` uses a pool, so we can't pass the moc connection directly.
732 // Instead, we're going to essentially recreate what `.record()` does calling
733 // the `.internal_record()` function, followed by the database `.record()`.
734 // This emulates
735 // ```
736 // pr.record(&Some(test_id.clone()), new, &None, Some(expr))
737 // .await?;
738 // ```
739
740 // and mock the redis call.
741 pr.internal_record(&mut conn, &old, new, Some(expr), &test_id)
742 .await?;
743
744 db_box
745 .log_report(&test_id, new)
746 .await
747 .inspect_err(|e| {
748 warn!("πβ οΈ Unable to record reliability state log: {:?}", e);
749 })
750 .map_err(|e| ApcErrorKind::GeneralError(e.to_string()))?;
751
752 Ok(())
753 }
754
755 #[actix_rt::test]
756 async fn test_push_reliability_record() -> Result<()> {
757 let db = crate::db::mock::MockDbClient::new();
758 let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
759 let new = ReliabilityState::Stored;
760 let old = ReliabilityState::Received;
761 let expr = 1;
762
763 let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
764 let new_key = ExpiryKey {
765 id: test_id.clone(),
766 state: new,
767 }
768 .to_string();
769 let old_key = ExpiryKey {
770 id: test_id.clone(),
771 state: old,
772 }
773 .to_string();
774
775 let state_key = format!("state.{test_id}");
776
777 let mut mock_pipe = redis::Pipeline::new();
778 mock_pipe
779 .cmd("MULTI")
780 .ignore()
781 // Decrement the old state
782 .cmd("HINCRBY")
783 .arg(COUNTS)
784 .arg(old.to_string())
785 .arg(-1)
786 .ignore()
787 // Replace the combined old state key in the expiry set.
788 .cmd("ZREM")
789 .arg(EXPIRY)
790 .arg(old_key)
791 .ignore()
792 .cmd("ZADD")
793 .arg(EXPIRY)
794 .arg(expr)
795 .arg(new_key)
796 .ignore()
797 .cmd("HINCRBY")
798 .arg(COUNTS)
799 .arg(new.to_string())
800 .arg(1)
801 .cmd("SET")
802 .arg(&state_key)
803 .arg(new.to_string())
804 .arg("XX")
805 .arg("GET")
806 .arg("KEEPTTL")
807 .ignore()
808 .cmd("EXEC")
809 .ignore();
810
811 let mut conn = MockRedisConnection::new(vec![
812 // Create the new state
813 MockCmd::new(
814 redis::cmd("WATCH").arg(&state_key).arg(EXPIRY),
815 Ok(redis::Value::Okay),
816 ),
817 MockCmd::new(
818 redis::cmd("GET").arg(&state_key),
819 Ok(redis::Value::SimpleString(old.to_string())),
820 ),
821 // NOTE: Technically, since we `.ignore()` these, we could just have a
822 // Vec containing just `Okay`. I'm being a bit pedantic here because I know
823 // that this will come back to haunt me if I'm not, and because figuring out
824 // the proper response for this was annoying.
825 MockCmd::new(
826 mock_pipe,
827 Ok(redis::Value::Array(vec![
828 redis::Value::Okay,
829 // Match the number of commands that are being held for processing
830 redis::Value::SimpleString("QUEUED".to_owned()), // multi
831 redis::Value::SimpleString("QUEUED".to_owned()), // hincrby
832 redis::Value::SimpleString("QUEUED".to_owned()), // zrem
833 redis::Value::SimpleString("QUEUED".to_owned()), // zadd
834 redis::Value::SimpleString("QUEUED".to_owned()), // hincrby
835 redis::Value::SimpleString("QUEUED".to_owned()), // set
836 // the exec has been called, return an array containing the results.
837 redis::Value::Array(vec![
838 redis::Value::Okay, //Multi
839 redis::Value::Okay, //hincrby
840 redis::Value::Okay, //zrem
841 redis::Value::Okay, //zadd
842 redis::Value::Okay, //hincr
843 redis::Value::SimpleString(old.to_string()),
844 ]),
845 ])),
846 ),
847 // If the transaction fails, this should return a redis::Value::Nil
848 MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
849 ]);
850
851 // test the main report function (note, this does not test redis)
852 let pr = PushReliability::new(
853 &None,
854 Box::new(Arc::new(db)),
855 &metrics,
856 crate::redis_util::MAX_TRANSACTION_LOOP,
857 )
858 .unwrap();
859 let _ = pr
860 .internal_record(&mut conn, &Some(old), new, Some(expr), &test_id)
861 .await;
862
863 Ok(())
864 }
865
866 #[actix_rt::test]
867 async fn test_push_reliability_full() -> Result<()> {
868 let db = crate::db::mock::MockDbClient::new();
869 let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
870 let new = ReliabilityState::Received;
871 let stored = ReliabilityState::Stored;
872 let expr = 1;
873
874 let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
875 let new_key = ExpiryKey {
876 id: test_id.clone(),
877 state: new,
878 }
879 .to_string();
880 let stored_key = ExpiryKey {
881 id: test_id.clone(),
882 state: stored,
883 }
884 .to_string();
885
886 let state_key = format!("state.{test_id}");
887
888 let mut mock_pipe = redis::Pipeline::new();
889 mock_pipe
890 .cmd("MULTI")
891 .ignore()
892 // Decrement the old state count
893 .cmd("HINCRBY")
894 .arg(COUNTS)
895 .arg(stored.to_string())
896 .arg(-1)
897 .ignore()
898 // Replace the expiry key
899 .cmd("ZREM")
900 .arg(EXPIRY)
901 .arg(stored_key.to_string())
902 .ignore()
903 .cmd("ZADD")
904 .arg(EXPIRY)
905 .arg(expr)
906 .arg(new_key)
907 .ignore()
908 // Increment the new state count
909 .cmd("HINCRBY")
910 .arg(COUNTS)
911 .arg(new.to_string())
912 .arg(1)
913 .ignore()
914 // And create the new state transition key (since the message is "live" again.)
915 .cmd("SET")
916 .arg(&state_key)
917 .arg(new.to_string())
918 .arg("XX")
919 .arg("GET")
920 .cmd("EXEC")
921 .ignore();
922
923 let mut conn = MockRedisConnection::new(vec![
924 MockCmd::new(
925 // Create the new state
926 redis::cmd("SET")
927 .arg(format!("state.{test_id}"))
928 .arg(new.to_string())
929 .arg("NX")
930 .arg("EX")
931 .arg(expr),
932 Ok(redis::Value::Okay),
933 ),
934 // increment the count for new
935 MockCmd::new(
936 redis::cmd("HINCRBY")
937 .arg(COUNTS)
938 .arg(new.to_string())
939 .arg(1),
940 Ok(redis::Value::Okay),
941 ),
942 // begin the transaction
943 MockCmd::new(
944 redis::cmd("WATCH").arg(&state_key).arg(EXPIRY),
945 Ok(redis::Value::Okay),
946 ),
947 // NOTE: Technically, since we `.ignore()` these, we could just have a
948 // Vec containing just `Okay`. I'm being a bit pedantic here because I know
949 // that this will come back to haunt me if I'm not, and because figuring out
950 // the proper response for this was annoying.
951 MockCmd::new(
952 mock_pipe,
953 Ok(redis::Value::Array(vec![
954 redis::Value::Okay,
955 // Match the number of commands that are being held for processing
956 redis::Value::SimpleString("QUEUED".to_owned()),
957 redis::Value::SimpleString("QUEUED".to_owned()),
958 redis::Value::SimpleString("QUEUED".to_owned()),
959 redis::Value::SimpleString("QUEUED".to_owned()),
960 // the exec has been called, return an array containing the results.
961 redis::Value::Array(vec![
962 redis::Value::Okay,
963 redis::Value::Okay,
964 redis::Value::Okay,
965 redis::Value::Okay,
966 ]),
967 ])),
968 ),
969 // If the transaction fails, this should return a redis::Value::Nil
970 MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
971 ]);
972
973 // test the main report function (note, this does not test redis)
974 let pr = PushReliability::new(
975 &None,
976 Box::new(Arc::new(db)),
977 &metrics,
978 crate::redis_util::MAX_TRANSACTION_LOOP,
979 )
980 .unwrap();
981 let _ = pr
982 .internal_record(&mut conn, &Some(stored), new, Some(expr), &test_id)
983 .await;
984
985 Ok(())
986 }
987
988 #[actix_rt::test]
989 async fn test_push_reliability_gc() -> Result<()> {
990 let db = crate::db::mock::MockDbClient::new();
991 let test_id = format!("TEST_VALUE_{}", Uuid::new_v4());
992 let new = ReliabilityState::Accepted;
993 let key = format!("{}#{}", &test_id, &new);
994 let expr = 1;
995 let metrics = Arc::new(StatsdClient::builder("", cadence::NopMetricSink).build());
996
997 let response: redis::Value = redis::Value::Array(vec![redis::Value::SimpleString(
998 ExpiryKey {
999 id: test_id.clone(),
1000 state: new,
1001 }
1002 .to_string(),
1003 )]);
1004
1005 // Construct the Pipeline.
1006 // A redis "pipeline" is a set of instructions that are executed in order. These
1007 // are not truly atomic, as other actions can interrupt a pipeline, but they
1008 // are guaranteed to happen in sequence. Transactions essentially check that the
1009 // WATCH key is not altered before the pipeline is executed.
1010 let mut mock_pipe = redis::Pipeline::new();
1011 mock_pipe
1012 .cmd("MULTI")
1013 .ignore()
1014 // Adjust the state counts.
1015 .cmd("HINCRBY")
1016 .arg(COUNTS)
1017 .arg(new.to_string())
1018 .arg(-1)
1019 .ignore()
1020 .cmd("HINCRBY")
1021 .arg(COUNTS)
1022 .arg(ReliabilityState::Expired.to_string())
1023 .arg(1)
1024 // Replace the state key in the expiry set.
1025 .ignore()
1026 .cmd("ZREM")
1027 .arg(EXPIRY)
1028 .arg(key)
1029 .ignore()
1030 .cmd("EXEC")
1031 .ignore();
1032
1033 // This mocks the "conn"ection, so all commands sent to the connection need
1034 // to be separated out. This includes the transaction "WATCH" and "UNWATCH".
1035 // The "meat" of the functions are handled via a Pipeline, which conjoins the
1036 // commands into one group (see above). Pipelines return an array of
1037 // results, one entry for each pipeline command.
1038 // In other words, the `internal_gc` commands are probably in the pipeline,
1039 // all the others are probably in the conn.
1040 let mut conn = MockRedisConnection::new(vec![
1041 MockCmd::new(redis::cmd("WATCH").arg(EXPIRY), Ok(redis::Value::Okay)),
1042 MockCmd::new(
1043 redis::cmd("ZRANGEBYSCORE").arg(EXPIRY).arg(0).arg(expr),
1044 Ok(response),
1045 ),
1046 // NOTE: Technically, since we `.ignore()` these, we could just have a
1047 // Vec containing just `Okay`. I'm being a bit pedantic here because I know
1048 // that this will come back to haunt me if I'm not, and because figuring out
1049 // the proper response for this was annoying.
1050 MockCmd::new(
1051 mock_pipe,
1052 Ok(redis::Value::Array(vec![
1053 redis::Value::Okay,
1054 // Match the number of commands that are being held for processing
1055 redis::Value::SimpleString("QUEUED".to_owned()),
1056 redis::Value::SimpleString("QUEUED".to_owned()),
1057 redis::Value::SimpleString("QUEUED".to_owned()),
1058 // the exec has been called, return an array containing the results.
1059 redis::Value::Array(vec![
1060 redis::Value::Okay,
1061 redis::Value::Okay,
1062 redis::Value::Okay,
1063 ]),
1064 ])),
1065 ),
1066 // If the transaction fails, this should return a redis::Value::Nil
1067 MockCmd::new(redis::cmd("UNWATCH"), Ok(redis::Value::Okay)),
1068 ]);
1069
1070 // test the main report function (note, this does not test redis)
1071 let pr = PushReliability::new(
1072 &None,
1073 Box::new(Arc::new(db)),
1074 &metrics,
1075 crate::redis_util::MAX_TRANSACTION_LOOP,
1076 )
1077 .unwrap();
1078 // functionally a no-op, but it does exercise lines.
1079 pr.gc().await?;
1080
1081 // and mock the redis call.
1082 pr.internal_gc(&mut conn, 1).await?;
1083
1084 Ok(())
1085 }
1086}