diff --git a/README.md b/README.md index 097895c..b059f1c 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,13 @@ Please refer to [Quickstart](https://github.com/lyc8503/UptimeFlare/wiki/Quickst - [x] Specify region for monitors - [x] TCP `opened` promise -- [x] Telegram example -- [x] [Bark](https://bark.day.app) example +- [x] Use apprise to support different notification channels +- [x] ~~Telegram example~~ +- [x] ~~[Bark](https://bark.day.app) example~~ +- [x] ~~Email notification via Cloudflare Email Workers~~ - [ ] Improve docs by providing simple examples - [ ] Notification grace period - [ ] SSL certificate checks - [ ] Self-host Dockerfile -- [ ] Email notification via Cloudflare Email Workers - [ ] Incident timeline +- [ ] Remove old incidents diff --git a/uptime.config.ts b/uptime.config.ts index f17fc80..b9c7636 100644 --- a/uptime.config.ts +++ b/uptime.config.ts @@ -58,6 +58,20 @@ const workerConfig = { timeout: 5000, }, ], + notification: { + // [Optional] apprise API server URL + // if not specified, no notification will be sent + appriseApiServer: "https://apprise.example.com/notify", + // [Optional] recipient URL for apprise, refer to https://github.com/caronc/apprise + // if not specified, no notification will be sent + recipientUrl: "tgram://bottoken/ChatID", + // [Optional] timezone used in notification messages, default to "Etc/GMT" + timeZone: "Asia/Shanghai", + // [Optional] grace period in minutes before sending a notification + // notification will be sent only if the monitor is down for N continuous checks after the initial failure + // if not specified, notification will be sent immediately + gracePeriod: 5, + }, callbacks: { onStatusChange: async ( env: any, @@ -70,8 +84,8 @@ const workerConfig = { // This callback will be called when there's a status change for any monitor // Write any Typescript code here - // By default, this sends Bark and Telegram notification on every status change if you setup Cloudflare env variables correctly. - await notify(env, monitor, isUp, timeIncidentStart, timeNow, reason) + // This will not follow the grace period settings and will be called immediately when the status changes + // You need to handle the grace period manually if you want to implement it }, onIncident: async ( env: any, @@ -86,117 +100,5 @@ const workerConfig = { }, } -// Below is code for sending Telegram & Bark notification -// You can safely ignore them -const escapeMarkdown = (text: string) => { - return text.replace(/[_*[\](){}~`>#+\-=|.!\\]/g, '\\$&'); -}; - -async function notify( - env: any, - monitor: any, - isUp: boolean, - timeIncidentStart: number, - timeNow: number, - reason: string, -) { - const dateFormatter = new Intl.DateTimeFormat('en-US', { - month: 'numeric', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - hour12: false, - timeZone: 'Asia/Shanghai', - }); - - let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60); - const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000)); - let statusText = isUp - ? `The service is up again after being down for ${downtimeDuration} minutes.` - : `Service became unavailable at ${timeIncidentStartFormatted}. Issue: ${reason || 'unspecified'}`; - - console.log('Notifying: ', monitor.name, statusText); - - if (env.BARK_SERVER && env.BARK_DEVICE_KEY) { - try { - let title = isUp ? `✅ ${monitor.name} is up again!` : `🔴 ${monitor.name} is currently down.`; - await sendBarkNotification(env, monitor, title, statusText); - } catch (error) { - console.error('Error sending Bark notification:', error); - } - } - - if (env.SECRET_TELEGRAM_CHAT_ID && env.SECRET_TELEGRAM_API_TOKEN) { - try { - let operationalLabel = isUp ? 'Up' : 'Down'; - let statusEmoji = isUp ? '✅' : '🔴'; - let telegramText = `*${escapeMarkdown( - monitor.name, - )}* is currently *${operationalLabel}*\n${statusEmoji} ${escapeMarkdown(statusText)}`; - await notifyTelegram(env, monitor, isUp, telegramText); - } catch (error) { - console.error('Error sending Telegram notification:', error); - } - } -} - -export async function notifyTelegram(env: any, monitor: any, operational: boolean, text: string) { - const chatId = env.SECRET_TELEGRAM_CHAT_ID; - const apiToken = env.SECRET_TELEGRAM_API_TOKEN; - - const payload = new URLSearchParams({ - chat_id: chatId, - parse_mode: 'MarkdownV2', - text: text, - }); - - try { - const response = await fetch(`https://api.telegram.org/bot${apiToken}/sendMessage`, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: payload.toString(), - }); - - if (!response.ok) { - console.error( - `Failed to send Telegram notification "${text}", ${response.status} ${response.statusText - } ${await response.text()}`, - ); - } - } catch (error) { - console.error('Error sending Telegram notification:', error); - } -} - -async function sendBarkNotification(env: any, monitor: any, title: string, body: string, group: string = '') { - const barkServer = env.BARK_SERVER; - const barkDeviceKey = env.BARK_DEVICE_KEY; - const barkUrl = `${barkServer}/push`; - const data = { - title: title, - body: body, - group: group, - url: monitor.url, - device_key: barkDeviceKey, - }; - - const response = await fetch(barkUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - - if (response.ok) { - console.log('Bark notification sent successfully.'); - } else { - const respText = await response.text(); - console.error('Failed to send Bark notification:', response.status, response.statusText, respText); - } -} - // Don't forget this, otherwise compilation fails. export { pageConfig, workerConfig } diff --git a/worker/src/index.ts b/worker/src/index.ts index 9fe08b7..cb79387 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -1,5 +1,5 @@ import { workerConfig } from '../../uptime.config' -import { getWorkerLocation } from './util' +import { formatStatusChangeNotification, getWorkerLocation, notifyWithApprise } from './util' import { MonitorState } from '../../uptime.types' import { getStatus } from './monitor' @@ -42,6 +42,34 @@ export default { const workerLocation = (await getWorkerLocation()) || 'ERROR' console.log(`Running scheduled event on ${workerLocation}...`) + // Auxiliary function to format notification and send it via apprise + let formatAndNotify = async ( + monitor: any, + isUp: boolean, + timeIncidentStart: number, + timeNow: number, + reason: string + ) => { + if (workerConfig.notification?.appriseApiServer && workerConfig.notification?.recipientUrl) { + const notification = formatStatusChangeNotification( + monitor, + isUp, + timeIncidentStart, + timeNow, + reason, + workerConfig.notification?.timeZone ?? 'Etc/GMT' + ) + await notifyWithApprise( + workerConfig.notification.appriseApiServer, + workerConfig.notification.recipientUrl, + notification.title, + notification.body + ) + } else { + console.log(`Apprise API server or recipient URL not set, skipping apprise notification for ${monitor.name}`) + } + } + // Read state, set init state if it doesn't exist let state = ((await env.UPTIMEFLARE_STATE.get('state', { @@ -63,10 +91,10 @@ export default { // Check each monitor // TODO: concurrent status check - for (const monitor of workerConfig.monitors) { console.log(`[${workerLocation}] Checking ${monitor.name}...`) + let monitorStatusChanged = false let checkLocation = workerLocation let status @@ -109,16 +137,33 @@ export default { }, ] // Then lastIncident here must not be undefined - const lastIncident = state.incident[monitor.id].slice(-1)[0] + let lastIncident = state.incident[monitor.id].slice(-1)[0] if (status.up) { // Current status is up // close existing incident if any if (lastIncident.end === undefined) { lastIncident.end = currentTimeSecond - statusChanged = true - + monitorStatusChanged = true try { + if ( + // grace period not set OR ... + workerConfig.notification?.gracePeriod === undefined || + // only when we have sent a notification for DOWN status, we will send a notification for UP status (within 30 seconds of possible drift) + currentTimeSecond - lastIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30 + ) { + await formatAndNotify( + monitor, + true, + lastIncident.start[0], + currentTimeSecond, + 'OK' + ) + } else { + console.log(`grace period (${workerConfig.notification?.gracePeriod}m) not met, skipping apprise UP notification for ${monitor.name}`) + } + + console.log('Calling config onStatusChange callback...') await workerConfig.callbacks.onStatusChange( env, monitor, @@ -141,21 +186,7 @@ export default { end: undefined, error: [status.err], }) - statusChanged = true - - try { - await workerConfig.callbacks.onStatusChange( - env, - monitor, - false, - currentTimeSecond, - currentTimeSecond, - status.err - ) - } catch (e) { - console.log('Error calling callback: ') - console.log(e) - } + monitorStatusChanged = true } else if ( lastIncident.end === undefined && lastIncident.error.slice(-1)[0] !== status.err @@ -163,28 +194,62 @@ export default { // append if the error message changes lastIncident.start.push(currentTimeSecond) lastIncident.error.push(status.err) - statusChanged = true + monitorStatusChanged = true + } - try { + const currentIncident = state.incident[monitor.id].slice(-1)[0] + try { + if ( + // monitor status changed AND... + (monitorStatusChanged && ( + // grace period not set OR ... + workerConfig.notification?.gracePeriod === undefined || + // have sent a notification for DOWN status + currentTimeSecond - currentIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30 + )) + || + ( + // grace period is set AND... + workerConfig.notification?.gracePeriod !== undefined && + ( + // grace period is met + currentTimeSecond - currentIncident.start[0] >= workerConfig.notification.gracePeriod * 60 - 30 && + currentTimeSecond - currentIncident.start[0] < workerConfig.notification.gracePeriod * 60 + 30 + ) + )) { + await formatAndNotify( + monitor, + false, + currentIncident.start[0], + currentTimeSecond, + status.err + ) + } else { + console.log(`Grace period (${workerConfig.notification?.gracePeriod}m) not met (currently down for ${currentTimeSecond - currentIncident.start[0]}s, changed ${monitorStatusChanged}), skipping apprise DOWN notification for ${monitor.name}`) + } + + if (monitorStatusChanged) { + console.log('Calling config onStatusChange callback...') await workerConfig.callbacks.onStatusChange( env, monitor, false, - lastIncident.start[0], + currentIncident.start[0], currentTimeSecond, status.err ) - } catch (e) { - console.log('Error calling callback: ') - console.log(e) } + } catch (e) { + console.log('Error calling callback: ') + console.log(e) } try { + console.log('Calling config onIncident callback...') await workerConfig.callbacks.onIncident( env, monitor, - lastIncident.start[0], + currentIncident.start[0], currentTimeSecond, status.err ) @@ -218,6 +283,9 @@ export default { latencyLists.all.shift() } state.latency[monitor.id] = latencyLists + // TODO: discard old incidents + + statusChanged ||= monitorStatusChanged } console.log(`statusChanged: ${statusChanged}, lastUpdate: ${state.lastUpdate}, currentTime: ${currentTimeSecond}`) diff --git a/worker/src/monitor.ts b/worker/src/monitor.ts index 1e6f8d4..216c06e 100644 --- a/worker/src/monitor.ts +++ b/worker/src/monitor.ts @@ -57,6 +57,7 @@ export async function getStatus( if (monitor.expectedCodes) { if (!monitor.expectedCodes.includes(response.status)) { + console.log(`${monitor.name} expected ${monitor.expectedCodes}, got ${response.status}`) status.up = false status.err = `Expected codes: ${JSON.stringify(monitor.expectedCodes)}, Got: ${response.status }` @@ -64,6 +65,7 @@ export async function getStatus( } } else { if (response.status < 200 || response.status > 299) { + console.log(`${monitor.name} expected 2xx, got ${response.status}`) status.up = false status.err = `Expected codes: 2xx, Got: ${response.status}` return status @@ -73,6 +75,7 @@ export async function getStatus( if (monitor.responseKeyword) { const responseBody = await response.text() if (!responseBody.includes(monitor.responseKeyword)) { + console.log(`${monitor.name} expected keyword ${monitor.responseKeyword}, not found in response (truncated to 100 chars): ${responseBody.slice(0, 100)}`) status.up = false status.err = "HTTP response doesn't contain the configured keyword" return status diff --git a/worker/src/util.ts b/worker/src/util.ts index 2de3f3b..8749555 100644 --- a/worker/src/util.ts +++ b/worker/src/util.ts @@ -1,29 +1,100 @@ async function getWorkerLocation() { - const res = await fetch('https://cloudflare.com/cdn-cgi/trace') - const text = await res.text() + const res = await fetch('https://cloudflare.com/cdn-cgi/trace') + const text = await res.text() - const colo = /^colo=(.*)$/m.exec(text)?.[1] - return colo + const colo = /^colo=(.*)$/m.exec(text)?.[1] + return colo } const fetchTimeout = ( - url: string, - ms: number, - { signal, ...options }: RequestInit | undefined = {} + url: string, + ms: number, + { signal, ...options }: RequestInit | undefined = {} ): Promise => { - const controller = new AbortController() - const promise = fetch(url, { signal: controller.signal, ...options }) - if (signal) signal.addEventListener('abort', () => controller.abort()) - const timeout = setTimeout(() => controller.abort(), ms) - return promise.finally(() => clearTimeout(timeout)) + const controller = new AbortController() + const promise = fetch(url, { signal: controller.signal, ...options }) + if (signal) signal.addEventListener('abort', () => controller.abort()) + const timeout = setTimeout(() => controller.abort(), ms) + return promise.finally(() => clearTimeout(timeout)) } function withTimeout(millis: number, promise: Promise): Promise { - const timeout = new Promise((resolve, reject) => - setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis) - ) + const timeout = new Promise((resolve, reject) => + setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis) + ) - return Promise.race([promise, timeout]) + return Promise.race([promise, timeout]) } -export { getWorkerLocation, fetchTimeout, withTimeout } +function formatStatusChangeNotification( + monitor: any, + isUp: boolean, + timeIncidentStart: number, + timeNow: number, + reason: string, + timeZone: string +) { + const dateFormatter = new Intl.DateTimeFormat('en-US', { + month: 'numeric', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hour12: false, + timeZone: timeZone, + }) + + let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60); + const timeNowFormatted = dateFormatter.format(new Date(timeNow * 1000)) + const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000)) + + if (isUp) { + return { + title: `✅ ${monitor.name} is up!`, + body: `The service is up again after being down for ${downtimeDuration} minutes.`, + } + } else if (timeNow == timeIncidentStart) { + return { + title: `🔴 ${monitor.name} is currently down.`, + body: `Service is unavailable at ${timeNowFormatted}. Issue: ${reason || 'unspecified'}`, + } + } else { + return { + title: `🔴 ${monitor.name} is still down.`, + body: `Service is unavailable since ${timeIncidentStartFormatted} (${downtimeDuration} minutes). Issue: ${reason || 'unspecified'}`, + } + } +} + +async function notifyWithApprise( + appriseApiServer: string, + recipientUrl: string, + title: string, + body: string +) { + console.log('Sending Apprise notification: ' + title + '-' + body + ' to ' + recipientUrl + ' via ' + appriseApiServer) + try { + const resp = await fetchTimeout(appriseApiServer, 5000, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + urls: recipientUrl, + title, + body, + type: 'warning', + format: 'text' + }), + }) + + if (!resp.ok) { + console.log('Error calling apprise server, code: ' + resp.status + ', response: ' + await resp.text()) + } else { + console.log('Apprise notification sent successfully, code: ' + resp.status) + } + } catch (e) { + console.log('Error calling apprise server: ' + e) + } +} + +export { getWorkerLocation, fetchTimeout, withTimeout, notifyWithApprise, formatStatusChangeNotification }