From 5c50da21071112b31223ad15e12b5f51f59e60ad Mon Sep 17 00:00:00 2001 From: Joe Abbate Date: Sat, 3 May 2025 12:45:01 -0400 Subject: [PATCH 1/3] Add Labels to Error Logs to Better Determine Failure Point --- src/worker.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/worker.rs b/src/worker.rs index cb7bf55..cf9d542 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -268,7 +268,7 @@ pub async fn work_loop( let job: Item = match queue.get_job().await { Ok(job) => job, Err(err) => { - error!("{}", err); + error!("Failed to Get Job: {}", err); continue; } }; @@ -279,11 +279,11 @@ pub async fn work_loop( } // Drop a job that should be retried - it will be returned to the work queue after // the (5 second) lease expires. - Err(err) if err.should_retry => error!("{}", err.msg), + Err(err) if err.should_retry => error!("Job Failed: {}, Retrying", err.msg), // Errors that shouldn't cause a retry should mark the job as complete so it isn't // tried again. Err(err) => { - error!("{}", err.msg); + error!("Job Failed: {}, Not Retrying", err.msg); queue.complete(&job).await?; } } From 13b11745d69dd9bc742e9e53cb326bd6954770ba Mon Sep 17 00:00:00 2001 From: Joe Abbate Date: Sat, 3 May 2025 13:08:28 -0400 Subject: [PATCH 2/3] Fail Worker Application After 3 Failures, 3 Seconds Apart --- src/worker.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/worker.rs b/src/worker.rs index cf9d542..36f3a8c 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,5 +1,4 @@ -use std::{collections::HashSet, env}; - +use std::{collections::HashSet, env, thread, time}; use anyhow::{anyhow, Result}; use log::error; use redis_work_queue::{Item, KeyPrefix, WorkQueue}; @@ -46,8 +45,7 @@ pub async fn main() -> Result<()> { env::var("PINGS_REMOVE_ROUTE").expect("PINGS_REMOVE_ROUTE must be set"), )?; - work_loop(queue, db_pool, pings).await?; - Ok(()) + work_loop(queue, db_pool, pings).await } async fn get_simple_data( @@ -263,15 +261,24 @@ pub async fn work_loop( db_pool: Pool, pings: PingClient, ) -> Result<()> { + let mut queue_connect_failure = 0; + let three_sec = time::Duration::from_secs(3); loop { // Wait for a job with no timeout and a lease time of 5 seconds. let job: Item = match queue.get_job().await { Ok(job) => job, Err(err) => { error!("Failed to Get Job: {}", err); + queue_connect_failure += 1; + thread::sleep(three_sec); + if queue_connect_failure >= 3 { + error!("Failed to Fetch Job 3+ Times! Failing..."); + return Err(anyhow!("Fetch Job Failed 3 Times. Is Redis Running?")); + } continue; } }; + queue_connect_failure = 0; match work(&job, &db_pool, &pings, &mut queue).await { // Mark successful jobs as complete Ok(()) => { From 4d5a91a6a733e58349c844b1c0091438a305cb28 Mon Sep 17 00:00:00 2001 From: Joe Abbate Date: Sat, 3 May 2025 13:18:39 -0400 Subject: [PATCH 3/3] Fmt --- src/worker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.rs b/src/worker.rs index 36f3a8c..e334f41 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,8 +1,8 @@ -use std::{collections::HashSet, env, thread, time}; use anyhow::{anyhow, Result}; use log::error; use redis_work_queue::{Item, KeyPrefix, WorkQueue}; use sqlx::{postgres::PgPoolOptions, Pool, Postgres}; +use std::{collections::HashSet, env, thread, time}; use crate::{ app::{RedisJob, SimpleRiderChange},