autoendpoint/
error.rs

1//! Error types and transformations
2
3use crate::headers::vapid::VapidError;
4use crate::routers::RouterError;
5use actix_web::{
6    HttpResponse, Result,
7    dev::ServiceResponse,
8    error::{JsonPayloadError, PayloadError, ResponseError},
9    http::StatusCode,
10    http::header::{CacheControl, CacheDirective},
11    middleware::ErrorHandlerResponse,
12};
13// Sentry uses the backtrace crate, not std::backtrace.
14use actix_http::header;
15use backtrace::Backtrace;
16use rand::RngExt;
17use serde::ser::SerializeMap;
18use serde::{Serialize, Serializer};
19use std::error::Error;
20use std::fmt::{self, Display};
21use thiserror::Error;
22use validator::{ValidationErrors, ValidationErrorsKind};
23
24use autopush_common::{db::error::DbError, errors::ReportableError};
25
26/// Common `Result` type.
27pub type ApiResult<T> = Result<T, ApiError>;
28
29/// A link for more info on the returned error
30const ERROR_URL: &str = "http://autopush.readthedocs.io/en/latest/http.html#error-codes";
31/// The base `Retry-After` period, in seconds, when no upstream supplied one.
32const RETRY_AFTER_PERIOD: u64 = 120; // retry after 2 minutes;
33/// How far either side of [`RETRY_AFTER_PERIOD`] a generated `Retry-After` may
34/// land. See [`jittered_retry_after`].
35const RETRY_AFTER_JITTER: u64 = 30;
36
37/// A `Retry-After`, in seconds, for when no upstream supplied one.
38///
39/// Jittered around [`RETRY_AFTER_PERIOD`] so that senders throttled in the same
40/// moment don't all retry in the same later moment, rebuilding the spike that
41/// throttled them. The jitter is symmetric, leaving the mean at the base
42/// period: spreading retries costs no additional delivery latency on average.
43fn jittered_retry_after() -> u64 {
44    rand::rng().random_range(
45        RETRY_AFTER_PERIOD - RETRY_AFTER_JITTER..=RETRY_AFTER_PERIOD + RETRY_AFTER_JITTER,
46    )
47}
48
49/// The main error type.
50#[derive(Debug)]
51pub struct ApiError {
52    pub kind: ApiErrorKind,
53    pub backtrace: Backtrace,
54    pub extras: Option<Vec<(String, String)>>,
55}
56
57impl ApiError {
58    /// Render a 404 response
59    // wrapper during the move. this should switch to autopush-common's impl.
60    pub fn render_404<B>(res: ServiceResponse<B>) -> Result<ErrorHandlerResponse<B>> {
61        //TODO: remove unwrap here.
62        Ok(autopush_common::errors::render_404(res).unwrap())
63    }
64}
65
66/// The possible errors this application could encounter
67#[derive(Debug, Error)]
68pub enum ApiErrorKind {
69    #[error(transparent)]
70    Io(#[from] std::io::Error),
71
72    #[error(transparent)]
73    Metrics(#[from] cadence::MetricError),
74
75    #[error(transparent)]
76    Validation(#[from] validator::ValidationErrors),
77
78    #[error(transparent)]
79    PayloadError(actix_web::Error),
80
81    #[error(transparent)]
82    VapidError(#[from] VapidError),
83
84    #[error(transparent)]
85    Router(#[from] RouterError),
86
87    #[error(transparent)]
88    Jwt(#[from] jsonwebtoken::errors::Error),
89
90    #[error(transparent)]
91    Serde(#[from] serde_json::Error),
92
93    #[error(transparent)]
94    ReqwestError(#[from] reqwest::Error),
95
96    #[error("Error while validating token")]
97    TokenHashValidation(#[source] openssl::error::ErrorStack),
98
99    #[error("Error while creating secret")]
100    RegistrationSecretHash(#[source] openssl::error::ErrorStack),
101
102    #[error("Error while creating endpoint URL: {0}")]
103    EndpointUrl(#[source] autopush_common::errors::ApcError),
104
105    #[error("Database error: {0}")]
106    Database(#[from] DbError),
107
108    #[error("Conditional database operation failed: {0}")]
109    Conditional(String),
110
111    #[error("Invalid token")]
112    InvalidToken,
113
114    #[error("UAID not found")]
115    NoUser,
116
117    #[error("No such subscription")]
118    NoSubscription,
119
120    /// A specific issue with the encryption headers
121    #[error("{0}")]
122    InvalidEncryption(String),
123
124    /// Used if the API version given is not v1 or v2
125    #[error("Invalid API version")]
126    InvalidApiVersion,
127
128    #[error("Missing TTL value")]
129    NoTTL,
130
131    #[error("Invalid router type")]
132    InvalidRouterType,
133
134    #[error("Invalid router token")]
135    InvalidRouterToken,
136
137    #[error("Invalid message ID")]
138    InvalidMessageId,
139
140    #[error("Invalid Authentication")]
141    InvalidAuthentication,
142
143    #[error("Invalid Local Auth {0}")]
144    InvalidLocalAuth(String),
145
146    #[error("General error {0}")]
147    General(String),
148
149    #[error("ERROR:Success")]
150    LogCheck,
151}
152
153impl ApiErrorKind {
154    /// The upstream-supplied `Retry-After`, in seconds, when a bridge sent one.
155    ///
156    /// `None` leaves the caller to its own default; see
157    /// [`jittered_retry_after`].
158    pub fn retry_after(&self) -> Option<u64> {
159        match self {
160            ApiErrorKind::Router(e) => e.retry_after(),
161            _ => None,
162        }
163    }
164
165    /// Get the associated HTTP status code
166    pub fn status(&self) -> StatusCode {
167        match self {
168            ApiErrorKind::PayloadError(e) => e.as_response_error().status_code(),
169            ApiErrorKind::Router(e) => e.status(),
170
171            ApiErrorKind::Validation(_)
172            | ApiErrorKind::InvalidEncryption(_)
173            | ApiErrorKind::NoTTL
174            | ApiErrorKind::InvalidRouterType
175            | ApiErrorKind::InvalidRouterToken
176            | ApiErrorKind::InvalidMessageId => StatusCode::BAD_REQUEST,
177
178            ApiErrorKind::VapidError(_)
179            | ApiErrorKind::Jwt(_)
180            | ApiErrorKind::Serde(_)
181            | ApiErrorKind::TokenHashValidation(_)
182            | ApiErrorKind::InvalidAuthentication
183            | ApiErrorKind::InvalidLocalAuth(_) => StatusCode::UNAUTHORIZED,
184
185            ApiErrorKind::InvalidToken | ApiErrorKind::InvalidApiVersion => StatusCode::NOT_FOUND,
186
187            ApiErrorKind::NoUser | ApiErrorKind::NoSubscription => StatusCode::GONE,
188
189            ApiErrorKind::LogCheck => StatusCode::IM_A_TEAPOT,
190
191            ApiErrorKind::Conditional(_) => StatusCode::SERVICE_UNAVAILABLE,
192
193            ApiErrorKind::Database(e) => e.status(),
194
195            ApiErrorKind::General(_)
196            | ApiErrorKind::Io(_)
197            | ApiErrorKind::Metrics(_)
198            | ApiErrorKind::EndpointUrl(_)
199            | ApiErrorKind::RegistrationSecretHash(_)
200            | ApiErrorKind::ReqwestError(_) => StatusCode::INTERNAL_SERVER_ERROR,
201        }
202    }
203
204    /// Specify the label to use for metrics reporting.
205    pub fn metric_label(&self) -> Option<&'static str> {
206        Some(match self {
207            ApiErrorKind::PayloadError(_) => "payload_error",
208            ApiErrorKind::Router(e) => return e.metric_label(),
209
210            ApiErrorKind::Validation(_) => "validation",
211            ApiErrorKind::InvalidEncryption(_) => "invalid_encryption",
212            ApiErrorKind::NoTTL => "no_ttl",
213            ApiErrorKind::InvalidRouterType => "invalid_router_type",
214            ApiErrorKind::InvalidRouterToken => "invalid_router_token",
215            ApiErrorKind::InvalidMessageId => "invalid_message_id",
216
217            ApiErrorKind::VapidError(_) => "vapid_error",
218            ApiErrorKind::Jwt(_) | ApiErrorKind::Serde(_) => "jwt",
219            ApiErrorKind::TokenHashValidation(_) => "token_hash_validation",
220            ApiErrorKind::InvalidAuthentication => "invalid_authentication",
221            ApiErrorKind::InvalidLocalAuth(_) => "invalid_local_auth",
222
223            ApiErrorKind::InvalidToken => "invalid_token",
224            ApiErrorKind::InvalidApiVersion => "invalid_api_version",
225
226            ApiErrorKind::NoUser => "no_user",
227            ApiErrorKind::NoSubscription => "no_subscription",
228
229            ApiErrorKind::LogCheck => "log_check",
230
231            ApiErrorKind::General(_) => "general",
232            ApiErrorKind::Io(_) => "io",
233            ApiErrorKind::Metrics(_) => "metrics",
234            ApiErrorKind::Database(e) => return e.metric_label(),
235            ApiErrorKind::Conditional(_) => "conditional",
236            ApiErrorKind::EndpointUrl(e) => return e.metric_label(),
237            ApiErrorKind::RegistrationSecretHash(_) => "registration_secret_hash",
238            ApiErrorKind::ReqwestError(_) => "reqwest",
239        })
240    }
241
242    /// Don't report all errors to sentry
243    pub fn is_sentry_event(&self) -> bool {
244        match self {
245            // ignore selected validation errors.
246            ApiErrorKind::Router(e) => e.is_sentry_event(),
247            ApiErrorKind::Database(e) => e.is_sentry_event(),
248            // Ignore common webpush errors
249            ApiErrorKind::NoTTL | ApiErrorKind::InvalidEncryption(_) |
250            // Ignore common VAPID erros
251            ApiErrorKind::VapidError(_)
252                | ApiErrorKind::Jwt(_)
253                | ApiErrorKind::TokenHashValidation(_)
254                | ApiErrorKind::InvalidAuthentication
255                | ApiErrorKind::InvalidLocalAuth(_) |
256            // Ignore missing or invalid user errors
257            ApiErrorKind::NoUser | ApiErrorKind::NoSubscription |
258            // Ignore oversized payload.
259            ApiErrorKind::PayloadError(_) |
260            ApiErrorKind::Validation(_) |
261            ApiErrorKind::Conditional(_) |
262            ApiErrorKind::ReqwestError(_) => false,
263            _ => true,
264        }
265    }
266
267    /// Get the associated error number
268    pub fn errno(&self) -> Option<usize> {
269        match self {
270            ApiErrorKind::Router(e) => e.errno(),
271
272            ApiErrorKind::Validation(e) => errno_from_validation_errors(e),
273
274            ApiErrorKind::InvalidToken | ApiErrorKind::InvalidApiVersion => Some(102),
275
276            ApiErrorKind::NoUser => Some(103),
277
278            ApiErrorKind::PayloadError(error)
279                if matches!(error.as_error(), Some(PayloadError::Overflow))
280                    || matches!(error.as_error(), Some(JsonPayloadError::Overflow { .. })) =>
281            {
282                Some(104)
283            }
284
285            ApiErrorKind::NoSubscription => Some(106),
286
287            ApiErrorKind::InvalidRouterType => Some(108),
288
289            ApiErrorKind::VapidError(_)
290            | ApiErrorKind::TokenHashValidation(_)
291            | ApiErrorKind::Jwt(_)
292            | ApiErrorKind::Serde(_)
293            | ApiErrorKind::InvalidAuthentication
294            | ApiErrorKind::InvalidLocalAuth(_) => Some(109),
295
296            ApiErrorKind::InvalidEncryption(_) => Some(110),
297
298            ApiErrorKind::NoTTL => Some(111),
299
300            ApiErrorKind::LogCheck => Some(999),
301
302            ApiErrorKind::General(_)
303            | ApiErrorKind::Io(_)
304            | ApiErrorKind::Metrics(_)
305            | ApiErrorKind::Database(_)
306            | ApiErrorKind::Conditional(_)
307            | ApiErrorKind::PayloadError(_)
308            | ApiErrorKind::InvalidRouterToken
309            | ApiErrorKind::RegistrationSecretHash(_)
310            | ApiErrorKind::EndpointUrl(_)
311            | ApiErrorKind::InvalidMessageId
312            | ApiErrorKind::ReqwestError(_) => None,
313        }
314    }
315}
316
317impl Display for ApiError {
318    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
319        self.kind.fmt(f)
320    }
321}
322
323impl Error for ApiError {
324    fn source(&self) -> Option<&(dyn Error + 'static)> {
325        self.kind.source()
326    }
327}
328
329// Forward From impls to ApiError from ApiErrorKind. Because From is reflexive,
330// this impl also takes care of From<ApiErrorKind>.
331impl<T> From<T> for ApiError
332where
333    ApiErrorKind: From<T>,
334{
335    fn from(item: T) -> Self {
336        ApiError {
337            kind: ApiErrorKind::from(item),
338            backtrace: Backtrace::new_unresolved(),
339            extras: None,
340        }
341    }
342}
343
344impl ResponseError for ApiError {
345    fn status_code(&self) -> StatusCode {
346        self.kind.status()
347    }
348
349    fn error_response(&self) -> HttpResponse {
350        let mut builder = HttpResponse::build(self.kind.status());
351
352        match self.status_code() {
353            StatusCode::GONE => {
354                builder.insert_header(CacheControl(vec![CacheDirective::MaxAge(86400)]));
355            }
356            StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE => {
357                let retry_after = self.kind.retry_after().unwrap_or_else(jittered_retry_after);
358                builder.insert_header((header::RETRY_AFTER, retry_after.to_string()));
359            }
360            _ => {}
361        }
362
363        builder.json(self)
364    }
365}
366
367impl Serialize for ApiError {
368    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
369    where
370        S: Serializer,
371    {
372        let status = self.kind.status();
373        let mut map = serializer.serialize_map(Some(5))?;
374
375        map.serialize_entry("code", &status.as_u16())?;
376        map.serialize_entry("errno", &self.kind.errno())?;
377        map.serialize_entry("error", &status.canonical_reason())?;
378        map.serialize_entry("message", &self.kind.to_string())?;
379        map.serialize_entry("more_info", ERROR_URL)?;
380        map.end()
381    }
382}
383
384impl ReportableError for ApiError {
385    fn reportable_source(&self) -> Option<&(dyn ReportableError + 'static)> {
386        match &self.kind {
387            ApiErrorKind::EndpointUrl(e) => Some(e),
388            ApiErrorKind::Database(e) => Some(e),
389            _ => None,
390        }
391    }
392
393    fn backtrace(&self) -> Option<&Backtrace> {
394        Some(&self.backtrace)
395    }
396
397    fn is_sentry_event(&self) -> bool {
398        self.kind.is_sentry_event()
399    }
400
401    fn metric_label(&self) -> Option<&'static str> {
402        self.kind.metric_label()
403    }
404
405    fn extras(&self) -> Vec<(&str, String)> {
406        let mut extras: Vec<(&str, String)> = match &self.extras {
407            Some(extras) => extras.iter().map(|e| (e.0.as_str(), e.1.clone())).collect(),
408            None => Default::default(),
409        };
410
411        match &self.kind {
412            ApiErrorKind::Router(e) => extras.extend(e.extras()),
413            ApiErrorKind::LogCheck => extras.extend(vec![("coffee", "Unsupported".to_owned())]),
414            _ => {}
415        };
416        extras
417    }
418}
419
420/// Get the error number from validation errors. If multiple errors are present,
421/// the first one with a valid error code is used.
422fn errno_from_validation_errors(e: &ValidationErrors) -> Option<usize> {
423    // Build an iterator over the error numbers, then get the first one
424    e.errors()
425        .values()
426        .flat_map(|error| match error {
427            ValidationErrorsKind::Struct(inner_errors) => {
428                Box::new(errno_from_validation_errors(inner_errors).into_iter())
429                    as Box<dyn Iterator<Item = usize>>
430            }
431            ValidationErrorsKind::List(indexed_errors) => Box::new(
432                indexed_errors
433                    .values()
434                    .filter_map(|errors| errno_from_validation_errors(errors)),
435            )
436                as Box<dyn Iterator<Item = usize>>,
437            ValidationErrorsKind::Field(errors) => {
438                Box::new(errors.iter().filter_map(|error| error.code.parse().ok()))
439                    as Box<dyn Iterator<Item = usize>>
440            }
441        })
442        .next()
443}
444
445#[cfg(test)]
446mod tests {
447    use actix_web::ResponseError;
448    use autopush_common::{db::error::DbError, sentry::event_from_error};
449    use std::collections::HashSet;
450
451    use crate::routers::RouterError;
452    use crate::routers::fcm::error::FcmError;
453
454    use super::{
455        ApiError, ApiErrorKind, RETRY_AFTER_JITTER, RETRY_AFTER_PERIOD, header,
456        jittered_retry_after,
457    };
458    use crate::error::ReportableError;
459
460    #[test]
461    fn sentry_event_with_extras() {
462        let dbe = DbError::Integrity("foo".to_owned(), Some("bar".to_owned()));
463        let e: ApiError = ApiErrorKind::Database(dbe).into();
464        let event = event_from_error(&e);
465        assert_eq!(event.exception.len(), 2);
466        assert_eq!(event.exception[0].ty, "Integrity");
467        assert_eq!(event.exception[1].ty, "ApiError");
468        assert_eq!(event.extra.get("row"), Some(&"bar".into()));
469    }
470
471    /// A generated `Retry-After` stays inside the jitter band, and varies
472    /// across calls so throttled senders don't retry in lockstep.
473    #[test]
474    fn jittered_retry_after_spreads_within_band() {
475        let band =
476            (RETRY_AFTER_PERIOD - RETRY_AFTER_JITTER)..=(RETRY_AFTER_PERIOD + RETRY_AFTER_JITTER);
477        let values: HashSet<u64> = (0..200).map(|_| jittered_retry_after()).collect();
478
479        for value in &values {
480            assert!(band.contains(value), "{value} outside {band:?}");
481        }
482        // 200 draws from a 61-wide band collide into one value with
483        // probability 61 * (1/61)^200, so this cannot flake in practice.
484        assert!(values.len() > 1, "no jitter applied");
485    }
486
487    /// An upstream-supplied `Retry-After` is forwarded verbatim, unjittered:
488    /// the bridge's instruction wins over our own guess.
489    #[test]
490    fn retry_after_header_prefers_upstream() {
491        let e: ApiError = ApiErrorKind::Router(RouterError::Fcm(FcmError::Upstream {
492            error_code: "RESOURCE_EXHAUSTED".to_owned(),
493            message: "quota".to_owned(),
494            retry_after: Some(45),
495        }))
496        .into();
497        let response = e.error_response();
498
499        assert_eq!(response.status().as_u16(), 429);
500        assert_eq!(
501            response.headers().get(header::RETRY_AFTER).unwrap(),
502            "45",
503            "upstream Retry-After should not be jittered"
504        );
505    }
506
507    /// Without an upstream value, the header falls back to a jittered default.
508    #[test]
509    fn retry_after_header_falls_back_to_jitter() {
510        let e: ApiError = ApiErrorKind::Router(RouterError::Fcm(FcmError::Upstream {
511            error_code: "UNAVAILABLE".to_owned(),
512            message: "try later".to_owned(),
513            retry_after: None,
514        }))
515        .into();
516        let response = e.error_response();
517
518        assert_eq!(response.status().as_u16(), 503);
519        let secs: u64 = response
520            .headers()
521            .get(header::RETRY_AFTER)
522            .expect("a Retry-After header")
523            .to_str()
524            .unwrap()
525            .parse()
526            .expect("delta-seconds");
527        assert!(
528            ((RETRY_AFTER_PERIOD - RETRY_AFTER_JITTER)..=(RETRY_AFTER_PERIOD + RETRY_AFTER_JITTER))
529                .contains(&secs),
530            "secs = {secs}"
531        );
532    }
533
534    /// Ensure that Pool error metric labels are specified and that they return a 503 status code.
535    #[cfg(feature = "bigtable")]
536    #[test]
537    fn test_label_for_metrics() {
538        // specifically test for a timeout on pool entry creation.
539        let e: ApiError = ApiErrorKind::Database(DbError::BTError(
540            autopush_common::db::bigtable::BigTableError::PoolTimeout(
541                deadpool::managed::TimeoutType::Create,
542            ),
543        ))
544        .into();
545
546        // Remember, `autoendpoint` is prefixed to this metric label.
547        assert_eq!(
548            e.kind.metric_label(),
549            Some("storage.bigtable.error.pool_timeout")
550        );
551
552        // "Retry-After" is applied on any 503 response (See ApiError::error_response)
553        assert_eq!(e.kind.status(), actix_http::StatusCode::SERVICE_UNAVAILABLE)
554    }
555
556    /// Ensure that extras set on a given error are included in the ApiError.extras() call.
557    #[tokio::test]
558    async fn pass_extras() {
559        let e = RouterError::NotFound;
560        let mut ae = ApiError::from(e);
561        ae.extras = Some([("foo".to_owned(), "bar".to_owned())].to_vec());
562
563        let aex: Vec<(&str, String)> = ae.extras();
564        assert!(aex.contains(&("foo", "bar".to_owned())));
565
566        let e = ApiErrorKind::LogCheck;
567        let mut ae = ApiError::from(e);
568        ae.extras = Some([("foo".to_owned(), "bar".to_owned())].to_vec());
569
570        let aex: Vec<(&str, String)> = ae.extras();
571        assert!(aex.contains(&("foo", "bar".to_owned())));
572        assert!(aex.contains(&("coffee", "Unsupported".to_owned())));
573    }
574}
autoendpoint/error.rs

autoendpoint/
error.rs
