use apprise for better notification & fix callback bug & improve log
lyc8503
parent
d827a7059e
commit
0eb03ebfff
|
|
@ -34,11 +34,13 @@ Please refer to [Quickstart](https://github.com/lyc8503/UptimeFlare/wiki/Quickst
|
|||
|
||||
- [x] Specify region for monitors
|
||||
- [x] TCP `opened` promise
|
||||
- [x] Telegram example
|
||||
- [x] [Bark](https://bark.day.app) example
|
||||
- [x] Use apprise to support different notification channels
|
||||
- [x] ~~Telegram example~~
|
||||
- [x] ~~[Bark](https://bark.day.app) example~~
|
||||
- [x] ~~Email notification via Cloudflare Email Workers~~
|
||||
- [ ] Improve docs by providing simple examples
|
||||
- [ ] Notification grace period
|
||||
- [ ] SSL certificate checks
|
||||
- [ ] Self-host Dockerfile
|
||||
- [ ] Email notification via Cloudflare Email Workers
|
||||
- [ ] Incident timeline
|
||||
- [ ] Remove old incidents
|
||||
|
|
|
|||
130
uptime.config.ts
130
uptime.config.ts
|
|
@ -58,6 +58,20 @@ const workerConfig = {
|
|||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
notification: {
|
||||
// [Optional] apprise API server URL
|
||||
// if not specified, no notification will be sent
|
||||
appriseApiServer: "https://apprise.example.com/notify",
|
||||
// [Optional] recipient URL for apprise, refer to https://github.com/caronc/apprise
|
||||
// if not specified, no notification will be sent
|
||||
recipientUrl: "tgram://bottoken/ChatID",
|
||||
// [Optional] timezone used in notification messages, default to "Etc/GMT"
|
||||
timeZone: "Asia/Shanghai",
|
||||
// [Optional] grace period in minutes before sending a notification
|
||||
// notification will be sent only if the monitor is down for N continuous checks after the initial failure
|
||||
// if not specified, notification will be sent immediately
|
||||
gracePeriod: 5,
|
||||
},
|
||||
callbacks: {
|
||||
onStatusChange: async (
|
||||
env: any,
|
||||
|
|
@ -70,8 +84,8 @@ const workerConfig = {
|
|||
// This callback will be called when there's a status change for any monitor
|
||||
// Write any Typescript code here
|
||||
|
||||
// By default, this sends Bark and Telegram notification on every status change if you setup Cloudflare env variables correctly.
|
||||
await notify(env, monitor, isUp, timeIncidentStart, timeNow, reason)
|
||||
// This will not follow the grace period settings and will be called immediately when the status changes
|
||||
// You need to handle the grace period manually if you want to implement it
|
||||
},
|
||||
onIncident: async (
|
||||
env: any,
|
||||
|
|
@ -86,117 +100,5 @@ const workerConfig = {
|
|||
},
|
||||
}
|
||||
|
||||
// Below is code for sending Telegram & Bark notification
|
||||
// You can safely ignore them
|
||||
const escapeMarkdown = (text: string) => {
|
||||
return text.replace(/[_*[\](){}~`>#+\-=|.!\\]/g, '\\$&');
|
||||
};
|
||||
|
||||
async function notify(
|
||||
env: any,
|
||||
monitor: any,
|
||||
isUp: boolean,
|
||||
timeIncidentStart: number,
|
||||
timeNow: number,
|
||||
reason: string,
|
||||
) {
|
||||
const dateFormatter = new Intl.DateTimeFormat('en-US', {
|
||||
month: 'numeric',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
timeZone: 'Asia/Shanghai',
|
||||
});
|
||||
|
||||
let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60);
|
||||
const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000));
|
||||
let statusText = isUp
|
||||
? `The service is up again after being down for ${downtimeDuration} minutes.`
|
||||
: `Service became unavailable at ${timeIncidentStartFormatted}. Issue: ${reason || 'unspecified'}`;
|
||||
|
||||
console.log('Notifying: ', monitor.name, statusText);
|
||||
|
||||
if (env.BARK_SERVER && env.BARK_DEVICE_KEY) {
|
||||
try {
|
||||
let title = isUp ? `✅ ${monitor.name} is up again!` : `🔴 ${monitor.name} is currently down.`;
|
||||
await sendBarkNotification(env, monitor, title, statusText);
|
||||
} catch (error) {
|
||||
console.error('Error sending Bark notification:', error);
|
||||
}
|
||||
}
|
||||
|
||||
if (env.SECRET_TELEGRAM_CHAT_ID && env.SECRET_TELEGRAM_API_TOKEN) {
|
||||
try {
|
||||
let operationalLabel = isUp ? 'Up' : 'Down';
|
||||
let statusEmoji = isUp ? '✅' : '🔴';
|
||||
let telegramText = `*${escapeMarkdown(
|
||||
monitor.name,
|
||||
)}* is currently *${operationalLabel}*\n${statusEmoji} ${escapeMarkdown(statusText)}`;
|
||||
await notifyTelegram(env, monitor, isUp, telegramText);
|
||||
} catch (error) {
|
||||
console.error('Error sending Telegram notification:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function notifyTelegram(env: any, monitor: any, operational: boolean, text: string) {
|
||||
const chatId = env.SECRET_TELEGRAM_CHAT_ID;
|
||||
const apiToken = env.SECRET_TELEGRAM_API_TOKEN;
|
||||
|
||||
const payload = new URLSearchParams({
|
||||
chat_id: chatId,
|
||||
parse_mode: 'MarkdownV2',
|
||||
text: text,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await fetch(`https://api.telegram.org/bot${apiToken}/sendMessage`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
},
|
||||
body: payload.toString(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(
|
||||
`Failed to send Telegram notification "${text}", ${response.status} ${response.statusText
|
||||
} ${await response.text()}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error sending Telegram notification:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendBarkNotification(env: any, monitor: any, title: string, body: string, group: string = '') {
|
||||
const barkServer = env.BARK_SERVER;
|
||||
const barkDeviceKey = env.BARK_DEVICE_KEY;
|
||||
const barkUrl = `${barkServer}/push`;
|
||||
const data = {
|
||||
title: title,
|
||||
body: body,
|
||||
group: group,
|
||||
url: monitor.url,
|
||||
device_key: barkDeviceKey,
|
||||
};
|
||||
|
||||
const response = await fetch(barkUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
console.log('Bark notification sent successfully.');
|
||||
} else {
|
||||
const respText = await response.text();
|
||||
console.error('Failed to send Bark notification:', response.status, response.statusText, respText);
|
||||
}
|
||||
}
|
||||
|
||||
// Don't forget this, otherwise compilation fails.
|
||||
export { pageConfig, workerConfig }
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { workerConfig } from '../../uptime.config'
|
||||
import { getWorkerLocation } from './util'
|
||||
import { formatStatusChangeNotification, getWorkerLocation, notifyWithApprise } from './util'
|
||||
import { MonitorState } from '../../uptime.types'
|
||||
import { getStatus } from './monitor'
|
||||
|
||||
|
|
@ -42,6 +42,34 @@ export default {
|
|||
const workerLocation = (await getWorkerLocation()) || 'ERROR'
|
||||
console.log(`Running scheduled event on ${workerLocation}...`)
|
||||
|
||||
// Auxiliary function to format notification and send it via apprise
|
||||
let formatAndNotify = async (
|
||||
monitor: any,
|
||||
isUp: boolean,
|
||||
timeIncidentStart: number,
|
||||
timeNow: number,
|
||||
reason: string
|
||||
) => {
|
||||
if (workerConfig.notification?.appriseApiServer && workerConfig.notification?.recipientUrl) {
|
||||
const notification = formatStatusChangeNotification(
|
||||
monitor,
|
||||
isUp,
|
||||
timeIncidentStart,
|
||||
timeNow,
|
||||
reason,
|
||||
workerConfig.notification?.timeZone ?? 'Etc/GMT'
|
||||
)
|
||||
await notifyWithApprise(
|
||||
workerConfig.notification.appriseApiServer,
|
||||
workerConfig.notification.recipientUrl,
|
||||
notification.title,
|
||||
notification.body
|
||||
)
|
||||
} else {
|
||||
console.log(`Apprise API server or recipient URL not set, skipping apprise notification for ${monitor.name}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Read state, set init state if it doesn't exist
|
||||
let state =
|
||||
((await env.UPTIMEFLARE_STATE.get('state', {
|
||||
|
|
@ -63,10 +91,10 @@ export default {
|
|||
|
||||
// Check each monitor
|
||||
// TODO: concurrent status check
|
||||
|
||||
for (const monitor of workerConfig.monitors) {
|
||||
console.log(`[${workerLocation}] Checking ${monitor.name}...`)
|
||||
|
||||
let monitorStatusChanged = false
|
||||
let checkLocation = workerLocation
|
||||
let status
|
||||
|
||||
|
|
@ -109,16 +137,33 @@ export default {
|
|||
},
|
||||
]
|
||||
// Then lastIncident here must not be undefined
|
||||
const lastIncident = state.incident[monitor.id].slice(-1)[0]
|
||||
let lastIncident = state.incident[monitor.id].slice(-1)[0]
|
||||
|
||||
if (status.up) {
|
||||
// Current status is up
|
||||
// close existing incident if any
|
||||
if (lastIncident.end === undefined) {
|
||||
lastIncident.end = currentTimeSecond
|
||||
statusChanged = true
|
||||
|
||||
monitorStatusChanged = true
|
||||
try {
|
||||
if (
|
||||
// grace period not set OR ...
|
||||
workerConfig.notification?.gracePeriod === undefined ||
|
||||
// only when we have sent a notification for DOWN status, we will send a notification for UP status (within 30 seconds of possible drift)
|
||||
currentTimeSecond - lastIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30
|
||||
) {
|
||||
await formatAndNotify(
|
||||
monitor,
|
||||
true,
|
||||
lastIncident.start[0],
|
||||
currentTimeSecond,
|
||||
'OK'
|
||||
)
|
||||
} else {
|
||||
console.log(`grace period (${workerConfig.notification?.gracePeriod}m) not met, skipping apprise UP notification for ${monitor.name}`)
|
||||
}
|
||||
|
||||
console.log('Calling config onStatusChange callback...')
|
||||
await workerConfig.callbacks.onStatusChange(
|
||||
env,
|
||||
monitor,
|
||||
|
|
@ -141,21 +186,7 @@ export default {
|
|||
end: undefined,
|
||||
error: [status.err],
|
||||
})
|
||||
statusChanged = true
|
||||
|
||||
try {
|
||||
await workerConfig.callbacks.onStatusChange(
|
||||
env,
|
||||
monitor,
|
||||
false,
|
||||
currentTimeSecond,
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
monitorStatusChanged = true
|
||||
} else if (
|
||||
lastIncident.end === undefined &&
|
||||
lastIncident.error.slice(-1)[0] !== status.err
|
||||
|
|
@ -163,28 +194,62 @@ export default {
|
|||
// append if the error message changes
|
||||
lastIncident.start.push(currentTimeSecond)
|
||||
lastIncident.error.push(status.err)
|
||||
statusChanged = true
|
||||
monitorStatusChanged = true
|
||||
}
|
||||
|
||||
try {
|
||||
const currentIncident = state.incident[monitor.id].slice(-1)[0]
|
||||
try {
|
||||
if (
|
||||
// monitor status changed AND...
|
||||
(monitorStatusChanged && (
|
||||
// grace period not set OR ...
|
||||
workerConfig.notification?.gracePeriod === undefined ||
|
||||
// have sent a notification for DOWN status
|
||||
currentTimeSecond - currentIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30
|
||||
))
|
||||
||
|
||||
(
|
||||
// grace period is set AND...
|
||||
workerConfig.notification?.gracePeriod !== undefined &&
|
||||
(
|
||||
// grace period is met
|
||||
currentTimeSecond - currentIncident.start[0] >= workerConfig.notification.gracePeriod * 60 - 30 &&
|
||||
currentTimeSecond - currentIncident.start[0] < workerConfig.notification.gracePeriod * 60 + 30
|
||||
)
|
||||
)) {
|
||||
await formatAndNotify(
|
||||
monitor,
|
||||
false,
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
} else {
|
||||
console.log(`Grace period (${workerConfig.notification?.gracePeriod}m) not met (currently down for ${currentTimeSecond - currentIncident.start[0]}s, changed ${monitorStatusChanged}), skipping apprise DOWN notification for ${monitor.name}`)
|
||||
}
|
||||
|
||||
if (monitorStatusChanged) {
|
||||
console.log('Calling config onStatusChange callback...')
|
||||
await workerConfig.callbacks.onStatusChange(
|
||||
env,
|
||||
monitor,
|
||||
false,
|
||||
lastIncident.start[0],
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('Calling config onIncident callback...')
|
||||
await workerConfig.callbacks.onIncident(
|
||||
env,
|
||||
monitor,
|
||||
lastIncident.start[0],
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
|
|
@ -218,6 +283,9 @@ export default {
|
|||
latencyLists.all.shift()
|
||||
}
|
||||
state.latency[monitor.id] = latencyLists
|
||||
// TODO: discard old incidents
|
||||
|
||||
statusChanged ||= monitorStatusChanged
|
||||
}
|
||||
|
||||
console.log(`statusChanged: ${statusChanged}, lastUpdate: ${state.lastUpdate}, currentTime: ${currentTimeSecond}`)
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ export async function getStatus(
|
|||
|
||||
if (monitor.expectedCodes) {
|
||||
if (!monitor.expectedCodes.includes(response.status)) {
|
||||
console.log(`${monitor.name} expected ${monitor.expectedCodes}, got ${response.status}`)
|
||||
status.up = false
|
||||
status.err = `Expected codes: ${JSON.stringify(monitor.expectedCodes)}, Got: ${response.status
|
||||
}`
|
||||
|
|
@ -64,6 +65,7 @@ export async function getStatus(
|
|||
}
|
||||
} else {
|
||||
if (response.status < 200 || response.status > 299) {
|
||||
console.log(`${monitor.name} expected 2xx, got ${response.status}`)
|
||||
status.up = false
|
||||
status.err = `Expected codes: 2xx, Got: ${response.status}`
|
||||
return status
|
||||
|
|
@ -73,6 +75,7 @@ export async function getStatus(
|
|||
if (monitor.responseKeyword) {
|
||||
const responseBody = await response.text()
|
||||
if (!responseBody.includes(monitor.responseKeyword)) {
|
||||
console.log(`${monitor.name} expected keyword ${monitor.responseKeyword}, not found in response (truncated to 100 chars): ${responseBody.slice(0, 100)}`)
|
||||
status.up = false
|
||||
status.err = "HTTP response doesn't contain the configured keyword"
|
||||
return status
|
||||
|
|
|
|||
|
|
@ -1,29 +1,100 @@
|
|||
async function getWorkerLocation() {
|
||||
const res = await fetch('https://cloudflare.com/cdn-cgi/trace')
|
||||
const text = await res.text()
|
||||
const res = await fetch('https://cloudflare.com/cdn-cgi/trace')
|
||||
const text = await res.text()
|
||||
|
||||
const colo = /^colo=(.*)$/m.exec(text)?.[1]
|
||||
return colo
|
||||
const colo = /^colo=(.*)$/m.exec(text)?.[1]
|
||||
return colo
|
||||
}
|
||||
|
||||
const fetchTimeout = (
|
||||
url: string,
|
||||
ms: number,
|
||||
{ signal, ...options }: RequestInit<RequestInitCfProperties> | undefined = {}
|
||||
url: string,
|
||||
ms: number,
|
||||
{ signal, ...options }: RequestInit<RequestInitCfProperties> | undefined = {}
|
||||
): Promise<Response> => {
|
||||
const controller = new AbortController()
|
||||
const promise = fetch(url, { signal: controller.signal, ...options })
|
||||
if (signal) signal.addEventListener('abort', () => controller.abort())
|
||||
const timeout = setTimeout(() => controller.abort(), ms)
|
||||
return promise.finally(() => clearTimeout(timeout))
|
||||
const controller = new AbortController()
|
||||
const promise = fetch(url, { signal: controller.signal, ...options })
|
||||
if (signal) signal.addEventListener('abort', () => controller.abort())
|
||||
const timeout = setTimeout(() => controller.abort(), ms)
|
||||
return promise.finally(() => clearTimeout(timeout))
|
||||
}
|
||||
|
||||
function withTimeout<T>(millis: number, promise: Promise<T>): Promise<T> {
|
||||
const timeout = new Promise<T>((resolve, reject) =>
|
||||
setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis)
|
||||
)
|
||||
const timeout = new Promise<T>((resolve, reject) =>
|
||||
setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis)
|
||||
)
|
||||
|
||||
return Promise.race([promise, timeout])
|
||||
return Promise.race([promise, timeout])
|
||||
}
|
||||
|
||||
export { getWorkerLocation, fetchTimeout, withTimeout }
|
||||
function formatStatusChangeNotification(
|
||||
monitor: any,
|
||||
isUp: boolean,
|
||||
timeIncidentStart: number,
|
||||
timeNow: number,
|
||||
reason: string,
|
||||
timeZone: string
|
||||
) {
|
||||
const dateFormatter = new Intl.DateTimeFormat('en-US', {
|
||||
month: 'numeric',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
timeZone: timeZone,
|
||||
})
|
||||
|
||||
let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60);
|
||||
const timeNowFormatted = dateFormatter.format(new Date(timeNow * 1000))
|
||||
const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000))
|
||||
|
||||
if (isUp) {
|
||||
return {
|
||||
title: `✅ ${monitor.name} is up!`,
|
||||
body: `The service is up again after being down for ${downtimeDuration} minutes.`,
|
||||
}
|
||||
} else if (timeNow == timeIncidentStart) {
|
||||
return {
|
||||
title: `🔴 ${monitor.name} is currently down.`,
|
||||
body: `Service is unavailable at ${timeNowFormatted}. Issue: ${reason || 'unspecified'}`,
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
title: `🔴 ${monitor.name} is still down.`,
|
||||
body: `Service is unavailable since ${timeIncidentStartFormatted} (${downtimeDuration} minutes). Issue: ${reason || 'unspecified'}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyWithApprise(
|
||||
appriseApiServer: string,
|
||||
recipientUrl: string,
|
||||
title: string,
|
||||
body: string
|
||||
) {
|
||||
console.log('Sending Apprise notification: ' + title + '-' + body + ' to ' + recipientUrl + ' via ' + appriseApiServer)
|
||||
try {
|
||||
const resp = await fetchTimeout(appriseApiServer, 5000, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
urls: recipientUrl,
|
||||
title,
|
||||
body,
|
||||
type: 'warning',
|
||||
format: 'text'
|
||||
}),
|
||||
})
|
||||
|
||||
if (!resp.ok) {
|
||||
console.log('Error calling apprise server, code: ' + resp.status + ', response: ' + await resp.text())
|
||||
} else {
|
||||
console.log('Apprise notification sent successfully, code: ' + resp.status)
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error calling apprise server: ' + e)
|
||||
}
|
||||
}
|
||||
|
||||
export { getWorkerLocation, fetchTimeout, withTimeout, notifyWithApprise, formatStatusChangeNotification }
|
||||
|
|
|
|||
Loading…
Reference in New Issue