diff --git a/types/config.ts b/types/config.ts index d6fc04c..434d214 100644 --- a/types/config.ts +++ b/types/config.ts @@ -45,6 +45,11 @@ export type MonitorTarget = { responseForbiddenKeyword?: string checkProxy?: string checkProxyFallback?: boolean + // [OPTIONAL] Instant retry on failure: number of extra attempts before reporting + // the monitor as down (default 0 = no retry). Filters out transient blips. + retries?: number + // [OPTIONAL] Delay in ms between a failed attempt and the retry (default 5000). + retryDelayMs?: number } export type WorkerConfig = { diff --git a/uptime.config.full.ts b/uptime.config.full.ts index e402ba7..0fdaa49 100644 --- a/uptime.config.full.ts +++ b/uptime.config.full.ts @@ -74,6 +74,11 @@ const workerConfig: WorkerConfig = { checkProxy: 'https://xxx.example.com OR worker://weur', // [OPTIONAL] if true, the check will fallback to local if the specified proxy is down checkProxyFallback: true, + // [OPTIONAL] instant retry on failure: number of extra attempts before reporting + // the monitor as down, to filter out transient blips (default 0 = no retry) + retries: 1, + // [OPTIONAL] delay in ms between a failed attempt and the retry (default 5000) + retryDelayMs: 5000, }, // Example TCP Monitor { diff --git a/worker/src/monitor.ts b/worker/src/monitor.ts index 8293483..e10d2b6 100644 --- a/worker/src/monitor.ts +++ b/worker/src/monitor.ts @@ -356,52 +356,68 @@ export async function getStatus( export async function doMonitor(monitor: MonitorTarget, defaultLocation: string, env: Env) { let checkLocation = defaultLocation - let status + let status: { ping: number; up: boolean; err: string } = { ping: 0, up: false, err: '' } - if (monitor.checkProxy) { - // Initiate a check using proxy (Geo-specific monitoring) - try { - console.log(`[${monitor.id}] Calling check proxy: ${monitor.checkProxy}`) - let resp - if (monitor.checkProxy.startsWith('worker://')) { - const doLoc = monitor.checkProxy.replace('worker://', '') - const doId = env.REMOTE_CHECKER_DO.idFromName(monitor.id) - const doStub = env.REMOTE_CHECKER_DO.get(doId, { - locationHint: doLoc as DurableObjectLocationHint, - }) - resp = await doStub.getLocationAndStatus(monitor) - try { - // Kill the DO instance after use, to avoid extra resource usage - await doStub.kill() - } catch (err) { - // An error here is expected, ignore it - } - } else if (monitor.checkProxy.startsWith('globalping://')) { - resp = await getStatusWithGlobalPing(monitor) - } else { - resp = await ( - await fetch(monitor.checkProxy, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(monitor), + // Instant retry: if a check fails, re-check before reporting the monitor as down, + // to filter out transient blips (false positives). Controlled per-monitor via + // `retries` / `retryDelayMs` (default 0 retries = previous behavior, no retry). + const maxAttempts = 1 + (monitor.retries ?? 0) + const retryDelayMs = monitor.retryDelayMs ?? 5000 + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + checkLocation = defaultLocation + + if (monitor.checkProxy) { + // Initiate a check using proxy (Geo-specific monitoring) + try { + console.log(`[${monitor.id}] Calling check proxy: ${monitor.checkProxy}`) + let resp + if (monitor.checkProxy.startsWith('worker://')) { + const doLoc = monitor.checkProxy.replace('worker://', '') + const doId = env.REMOTE_CHECKER_DO.idFromName(monitor.id) + const doStub = env.REMOTE_CHECKER_DO.get(doId, { + locationHint: doLoc as DurableObjectLocationHint, }) - ).json<{ location: string; status: { ping: number; up: boolean; err: string } }>() - } - checkLocation = resp.location - status = resp.status - } catch (err) { - console.log(`[${monitor.id}] Error calling proxy: ${err}`) - if (monitor.checkProxyFallback) { - console.log('Falling back to local check...') - status = await getStatus(monitor) - } else { - // TODO: more consistent error handling (throw or return?) - status = { ping: 0, up: false, err: 'Unknown check proxy error' } + resp = await doStub.getLocationAndStatus(monitor) + try { + // Kill the DO instance after use, to avoid extra resource usage + await doStub.kill() + } catch (err) { + // An error here is expected, ignore it + } + } else if (monitor.checkProxy.startsWith('globalping://')) { + resp = await getStatusWithGlobalPing(monitor) + } else { + resp = await ( + await fetch(monitor.checkProxy, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(monitor), + }) + ).json<{ location: string; status: { ping: number; up: boolean; err: string } }>() + } + checkLocation = resp.location + status = resp.status + } catch (err) { + console.log(`[${monitor.id}] Error calling proxy: ${err}`) + if (monitor.checkProxyFallback) { + console.log('Falling back to local check...') + status = await getStatus(monitor) + } else { + // TODO: more consistent error handling (throw or return?) + status = { ping: 0, up: false, err: 'Unknown check proxy error' } + } } + } else { + // Initiate a check from the current location + status = await getStatus(monitor) } - } else { - // Initiate a check from the current location - status = await getStatus(monitor) + + if (status.up || attempt === maxAttempts) break + console.log( + `[${monitor.id}] Attempt ${attempt}/${maxAttempts} failed (${status.err}); retrying in ${retryDelayMs}ms...` + ) + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)) } console.log(`[${monitor.id}] Check result from ${checkLocation}: up=${status.up}, ping=${status.ping}, err=${status.err}`)